aboutsummaryrefslogtreecommitdiffstats
path: root/tools/lib
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lib')
-rw-r--r--tools/lib/bitmap.c6
-rw-r--r--tools/lib/bpf/Build2
-rw-r--r--tools/lib/bpf/Makefile2
-rw-r--r--tools/lib/bpf/bpf.c209
-rw-r--r--tools/lib/bpf/bpf.h109
-rw-r--r--tools/lib/bpf/bpf_core_read.h11
-rw-r--r--tools/lib/bpf/bpf_helpers.h13
-rw-r--r--tools/lib/bpf/bpf_tracing.h60
-rw-r--r--tools/lib/bpf/btf.c412
-rw-r--r--tools/lib/bpf/btf.h118
-rw-r--r--tools/lib/bpf/btf_dump.c160
-rw-r--r--tools/lib/bpf/gen_loader.c2
-rw-r--r--tools/lib/bpf/libbpf.c2222
-rw-r--r--tools/lib/bpf/libbpf.h569
-rw-r--r--tools/lib/bpf/libbpf.map123
-rw-r--r--tools/lib/bpf/libbpf_common.h16
-rw-r--r--tools/lib/bpf/libbpf_internal.h39
-rw-r--r--tools/lib/bpf/libbpf_legacy.h28
-rw-r--r--tools/lib/bpf/libbpf_probes.c125
-rw-r--r--tools/lib/bpf/linker.c7
-rw-r--r--tools/lib/bpf/netlink.c62
-rw-r--r--tools/lib/bpf/relo_core.c479
-rw-r--r--tools/lib/bpf/relo_core.h10
-rw-r--r--tools/lib/bpf/skel_internal.h4
-rw-r--r--tools/lib/bpf/usdt.bpf.h16
-rw-r--r--tools/lib/bpf/usdt.c129
-rw-r--r--tools/lib/bpf/xsk.c1260
-rw-r--r--tools/lib/bpf/xsk.h336
-rw-r--r--tools/lib/perf/cpumap.c2
-rw-r--r--tools/lib/perf/evlist.c50
-rw-r--r--tools/lib/perf/evsel.c79
-rw-r--r--tools/lib/perf/include/internal/evsel.h4
-rw-r--r--tools/lib/perf/include/perf/cpumap.h2
-rw-r--r--tools/lib/perf/include/perf/event.h58
-rw-r--r--tools/lib/perf/include/perf/evsel.h4
-rw-r--r--tools/lib/perf/tests/test-evsel.c161
36 files changed, 2233 insertions, 4656 deletions
diff --git a/tools/lib/bitmap.c b/tools/lib/bitmap.c
index 354f8cdc0880..c3e4871967bc 100644
--- a/tools/lib/bitmap.c
+++ b/tools/lib/bitmap.c
@@ -5,9 +5,9 @@
*/
#include <linux/bitmap.h>
-int __bitmap_weight(const unsigned long *bitmap, int bits)
+unsigned int __bitmap_weight(const unsigned long *bitmap, int bits)
{
- int k, w = 0, lim = bits/BITS_PER_LONG;
+ unsigned int k, w = 0, lim = bits/BITS_PER_LONG;
for (k = 0; k < lim; k++)
w += hweight_long(bitmap[k]);
@@ -57,7 +57,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits,
return ret;
}
-int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int bits)
{
unsigned int k;
diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build
index 31a1a9015902..5a3dfb56d78f 100644
--- a/tools/lib/bpf/Build
+++ b/tools/lib/bpf/Build
@@ -1,4 +1,4 @@
libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o \
- netlink.o bpf_prog_linfo.o libbpf_probes.o xsk.o hashmap.o \
+ netlink.o bpf_prog_linfo.o libbpf_probes.o hashmap.o \
btf_dump.o ringbuf.o strset.o linker.o gen_loader.o relo_core.o \
usdt.o
diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile
index a1265b152027..4c904ef0b47e 100644
--- a/tools/lib/bpf/Makefile
+++ b/tools/lib/bpf/Makefile
@@ -237,7 +237,7 @@ install_lib: all_cmd
$(call do_install_mkdir,$(libdir_SQ)); \
cp -fpR $(LIB_FILE) $(DESTDIR)$(libdir_SQ)
-SRC_HDRS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h xsk.h \
+SRC_HDRS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h \
bpf_helpers.h bpf_tracing.h bpf_endian.h bpf_core_read.h \
skel_internal.h libbpf_version.h usdt.bpf.h
GEN_HDRS := $(BPF_GENERATED)
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index 240186aac8e6..efcc06dafbd9 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -147,10 +147,6 @@ int bump_rlimit_memlock(void)
{
struct rlimit rlim;
- /* this the default in libbpf 1.0, but for now user has to opt-in explicitly */
- if (!(libbpf_mode & LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK))
- return 0;
-
/* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */
if (memlock_bumped || kernel_supports(NULL, FEAT_MEMCG_ACCOUNT))
return 0;
@@ -233,11 +229,10 @@ alloc_zero_tailing_info(const void *orecord, __u32 cnt,
return info;
}
-DEFAULT_VERSION(bpf_prog_load_v0_6_0, bpf_prog_load, LIBBPF_0.6.0)
-int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type,
- const char *prog_name, const char *license,
- const struct bpf_insn *insns, size_t insn_cnt,
- const struct bpf_prog_load_opts *opts)
+int bpf_prog_load(enum bpf_prog_type prog_type,
+ const char *prog_name, const char *license,
+ const struct bpf_insn *insns, size_t insn_cnt,
+ const struct bpf_prog_load_opts *opts)
{
void *finfo = NULL, *linfo = NULL;
const char *func_info, *line_info;
@@ -384,94 +379,6 @@ done:
return libbpf_err_errno(fd);
}
-__attribute__((alias("bpf_load_program_xattr2")))
-int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
- char *log_buf, size_t log_buf_sz);
-
-static int bpf_load_program_xattr2(const struct bpf_load_program_attr *load_attr,
- char *log_buf, size_t log_buf_sz)
-{
- LIBBPF_OPTS(bpf_prog_load_opts, p);
-
- if (!load_attr || !log_buf != !log_buf_sz)
- return libbpf_err(-EINVAL);
-
- p.expected_attach_type = load_attr->expected_attach_type;
- switch (load_attr->prog_type) {
- case BPF_PROG_TYPE_STRUCT_OPS:
- case BPF_PROG_TYPE_LSM:
- p.attach_btf_id = load_attr->attach_btf_id;
- break;
- case BPF_PROG_TYPE_TRACING:
- case BPF_PROG_TYPE_EXT:
- p.attach_btf_id = load_attr->attach_btf_id;
- p.attach_prog_fd = load_attr->attach_prog_fd;
- break;
- default:
- p.prog_ifindex = load_attr->prog_ifindex;
- p.kern_version = load_attr->kern_version;
- }
- p.log_level = load_attr->log_level;
- p.log_buf = log_buf;
- p.log_size = log_buf_sz;
- p.prog_btf_fd = load_attr->prog_btf_fd;
- p.func_info_rec_size = load_attr->func_info_rec_size;
- p.func_info_cnt = load_attr->func_info_cnt;
- p.func_info = load_attr->func_info;
- p.line_info_rec_size = load_attr->line_info_rec_size;
- p.line_info_cnt = load_attr->line_info_cnt;
- p.line_info = load_attr->line_info;
- p.prog_flags = load_attr->prog_flags;
-
- return bpf_prog_load(load_attr->prog_type, load_attr->name, load_attr->license,
- load_attr->insns, load_attr->insns_cnt, &p);
-}
-
-int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
- size_t insns_cnt, const char *license,
- __u32 kern_version, char *log_buf,
- size_t log_buf_sz)
-{
- struct bpf_load_program_attr load_attr;
-
- memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
- load_attr.prog_type = type;
- load_attr.expected_attach_type = 0;
- load_attr.name = NULL;
- load_attr.insns = insns;
- load_attr.insns_cnt = insns_cnt;
- load_attr.license = license;
- load_attr.kern_version = kern_version;
-
- return bpf_load_program_xattr2(&load_attr, log_buf, log_buf_sz);
-}
-
-int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns,
- size_t insns_cnt, __u32 prog_flags, const char *license,
- __u32 kern_version, char *log_buf, size_t log_buf_sz,
- int log_level)
-{
- union bpf_attr attr;
- int fd;
-
- bump_rlimit_memlock();
-
- memset(&attr, 0, sizeof(attr));
- attr.prog_type = type;
- attr.insn_cnt = (__u32)insns_cnt;
- attr.insns = ptr_to_u64(insns);
- attr.license = ptr_to_u64(license);
- attr.log_buf = ptr_to_u64(log_buf);
- attr.log_size = log_buf_sz;
- attr.log_level = log_level;
- log_buf[0] = 0;
- attr.kern_version = kern_version;
- attr.prog_flags = prog_flags;
-
- fd = sys_bpf_prog_load(&attr, sizeof(attr), PROG_LOAD_ATTEMPTS);
- return libbpf_err_errno(fd);
-}
-
int bpf_map_update_elem(int fd, const void *key, const void *value,
__u64 flags)
{
@@ -672,11 +579,20 @@ int bpf_obj_pin(int fd, const char *pathname)
int bpf_obj_get(const char *pathname)
{
+ return bpf_obj_get_opts(pathname, NULL);
+}
+
+int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts)
+{
union bpf_attr attr;
int fd;
+ if (!OPTS_VALID(opts, bpf_obj_get_opts))
+ return libbpf_err(-EINVAL);
+
memset(&attr, 0, sizeof(attr));
attr.pathname = ptr_to_u64((void *)pathname);
+ attr.file_flags = OPTS_GET(opts, file_flags, 0);
fd = sys_bpf_fd(BPF_OBJ_GET, &attr, sizeof(attr));
return libbpf_err_errno(fd);
@@ -888,80 +804,48 @@ int bpf_iter_create(int link_fd)
return libbpf_err_errno(fd);
}
-int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
- __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt)
+int bpf_prog_query_opts(int target_fd,
+ enum bpf_attach_type type,
+ struct bpf_prog_query_opts *opts)
{
union bpf_attr attr;
int ret;
+ if (!OPTS_VALID(opts, bpf_prog_query_opts))
+ return libbpf_err(-EINVAL);
+
memset(&attr, 0, sizeof(attr));
+
attr.query.target_fd = target_fd;
attr.query.attach_type = type;
- attr.query.query_flags = query_flags;
- attr.query.prog_cnt = *prog_cnt;
- attr.query.prog_ids = ptr_to_u64(prog_ids);
+ attr.query.query_flags = OPTS_GET(opts, query_flags, 0);
+ attr.query.prog_cnt = OPTS_GET(opts, prog_cnt, 0);
+ attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL));
+ attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL));
ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr));
- if (attach_flags)
- *attach_flags = attr.query.attach_flags;
- *prog_cnt = attr.query.prog_cnt;
-
- return libbpf_err_errno(ret);
-}
-
-int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size,
- void *data_out, __u32 *size_out, __u32 *retval,
- __u32 *duration)
-{
- union bpf_attr attr;
- int ret;
-
- memset(&attr, 0, sizeof(attr));
- attr.test.prog_fd = prog_fd;
- attr.test.data_in = ptr_to_u64(data);
- attr.test.data_out = ptr_to_u64(data_out);
- attr.test.data_size_in = size;
- attr.test.repeat = repeat;
-
- ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
-
- if (size_out)
- *size_out = attr.test.data_size_out;
- if (retval)
- *retval = attr.test.retval;
- if (duration)
- *duration = attr.test.duration;
+ OPTS_SET(opts, attach_flags, attr.query.attach_flags);
+ OPTS_SET(opts, prog_cnt, attr.query.prog_cnt);
return libbpf_err_errno(ret);
}
-int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr)
+int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
+ __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt)
{
- union bpf_attr attr;
+ LIBBPF_OPTS(bpf_prog_query_opts, opts);
int ret;
- if (!test_attr->data_out && test_attr->data_size_out > 0)
- return libbpf_err(-EINVAL);
+ opts.query_flags = query_flags;
+ opts.prog_ids = prog_ids;
+ opts.prog_cnt = *prog_cnt;
- memset(&attr, 0, sizeof(attr));
- attr.test.prog_fd = test_attr->prog_fd;
- attr.test.data_in = ptr_to_u64(test_attr->data_in);
- attr.test.data_out = ptr_to_u64(test_attr->data_out);
- attr.test.data_size_in = test_attr->data_size_in;
- attr.test.data_size_out = test_attr->data_size_out;
- attr.test.ctx_in = ptr_to_u64(test_attr->ctx_in);
- attr.test.ctx_out = ptr_to_u64(test_attr->ctx_out);
- attr.test.ctx_size_in = test_attr->ctx_size_in;
- attr.test.ctx_size_out = test_attr->ctx_size_out;
- attr.test.repeat = test_attr->repeat;
+ ret = bpf_prog_query_opts(target_fd, type, &opts);
- ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
-
- test_attr->data_size_out = attr.test.data_size_out;
- test_attr->ctx_size_out = attr.test.ctx_size_out;
- test_attr->retval = attr.test.retval;
- test_attr->duration = attr.test.duration;
+ if (attach_flags)
+ *attach_flags = opts.attach_flags;
+ *prog_cnt = opts.prog_cnt;
return libbpf_err_errno(ret);
}
@@ -1162,27 +1046,6 @@ int bpf_btf_load(const void *btf_data, size_t btf_size, const struct bpf_btf_loa
return libbpf_err_errno(fd);
}
-int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, bool do_log)
-{
- LIBBPF_OPTS(bpf_btf_load_opts, opts);
- int fd;
-
-retry:
- if (do_log && log_buf && log_buf_size) {
- opts.log_buf = log_buf;
- opts.log_size = log_buf_size;
- opts.log_level = 1;
- }
-
- fd = bpf_btf_load(btf, btf_size, &opts);
- if (fd < 0 && !do_log && log_buf && log_buf_size) {
- do_log = true;
- goto retry;
- }
-
- return libbpf_err_errno(fd);
-}
-
int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len,
__u32 *prog_id, __u32 *fd_type, __u64 *probe_offset,
__u64 *probe_addr)
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index cabc03703e29..9c50beabdd14 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -103,54 +103,6 @@ LIBBPF_API int bpf_prog_load(enum bpf_prog_type prog_type,
const char *prog_name, const char *license,
const struct bpf_insn *insns, size_t insn_cnt,
const struct bpf_prog_load_opts *opts);
-/* this "specialization" should go away in libbpf 1.0 */
-LIBBPF_API int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type,
- const char *prog_name, const char *license,
- const struct bpf_insn *insns, size_t insn_cnt,
- const struct bpf_prog_load_opts *opts);
-
-/* This is an elaborate way to not conflict with deprecated bpf_prog_load()
- * API, defined in libbpf.h. Once we hit libbpf 1.0, all this will be gone.
- * With this approach, if someone is calling bpf_prog_load() with
- * 4 arguments, they will use the deprecated API, which keeps backwards
- * compatibility (both source code and binary). If bpf_prog_load() is called
- * with 6 arguments, though, it gets redirected to __bpf_prog_load.
- * So looking forward to libbpf 1.0 when this hack will be gone and
- * __bpf_prog_load() will be called just bpf_prog_load().
- */
-#ifndef bpf_prog_load
-#define bpf_prog_load(...) ___libbpf_overload(___bpf_prog_load, __VA_ARGS__)
-#define ___bpf_prog_load4(file, type, pobj, prog_fd) \
- bpf_prog_load_deprecated(file, type, pobj, prog_fd)
-#define ___bpf_prog_load6(prog_type, prog_name, license, insns, insn_cnt, opts) \
- bpf_prog_load(prog_type, prog_name, license, insns, insn_cnt, opts)
-#endif /* bpf_prog_load */
-
-struct bpf_load_program_attr {
- enum bpf_prog_type prog_type;
- enum bpf_attach_type expected_attach_type;
- const char *name;
- const struct bpf_insn *insns;
- size_t insns_cnt;
- const char *license;
- union {
- __u32 kern_version;
- __u32 attach_prog_fd;
- };
- union {
- __u32 prog_ifindex;
- __u32 attach_btf_id;
- };
- __u32 prog_btf_fd;
- __u32 func_info_rec_size;
- const void *func_info;
- __u32 func_info_cnt;
- __u32 line_info_rec_size;
- const void *line_info;
- __u32 line_info_cnt;
- __u32 log_level;
- __u32 prog_flags;
-};
/* Flags to direct loading requirements */
#define MAPS_RELAX_COMPAT 0x01
@@ -158,22 +110,6 @@ struct bpf_load_program_attr {
/* Recommended log buffer size */
#define BPF_LOG_BUF_SIZE (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead")
-LIBBPF_API int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr,
- char *log_buf, size_t log_buf_sz);
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead")
-LIBBPF_API int bpf_load_program(enum bpf_prog_type type,
- const struct bpf_insn *insns, size_t insns_cnt,
- const char *license, __u32 kern_version,
- char *log_buf, size_t log_buf_sz);
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead")
-LIBBPF_API int bpf_verify_program(enum bpf_prog_type type,
- const struct bpf_insn *insns,
- size_t insns_cnt, __u32 prog_flags,
- const char *license, __u32 kern_version,
- char *log_buf, size_t log_buf_sz,
- int log_level);
-
struct bpf_btf_load_opts {
size_t sz; /* size of this struct for forward/backward compatibility */
@@ -187,10 +123,6 @@ struct bpf_btf_load_opts {
LIBBPF_API int bpf_btf_load(const void *btf_data, size_t btf_size,
const struct bpf_btf_load_opts *opts);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_btf_load() instead")
-LIBBPF_API int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf,
- __u32 log_buf_size, bool do_log);
-
LIBBPF_API int bpf_map_update_elem(int fd, const void *key, const void *value,
__u64 flags);
@@ -338,8 +270,19 @@ LIBBPF_API int bpf_map_update_batch(int fd, const void *keys, const void *values
__u32 *count,
const struct bpf_map_batch_opts *opts);
+struct bpf_obj_get_opts {
+ size_t sz; /* size of this struct for forward/backward compatibility */
+
+ __u32 file_flags;
+
+ size_t :0;
+};
+#define bpf_obj_get_opts__last_field file_flags
+
LIBBPF_API int bpf_obj_pin(int fd, const char *pathname);
LIBBPF_API int bpf_obj_get(const char *pathname);
+LIBBPF_API int bpf_obj_get_opts(const char *pathname,
+ const struct bpf_obj_get_opts *opts);
struct bpf_prog_attach_opts {
size_t sz; /* size of this struct for forward/backward compatibility */
@@ -353,10 +296,6 @@ LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd,
LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int attachable_fd,
enum bpf_attach_type type,
const struct bpf_prog_attach_opts *opts);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_prog_attach_opts() instead")
-LIBBPF_API int bpf_prog_attach_xattr(int prog_fd, int attachable_fd,
- enum bpf_attach_type type,
- const struct bpf_prog_attach_opts *opts);
LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type);
LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd,
enum bpf_attach_type type);
@@ -422,17 +361,6 @@ struct bpf_prog_test_run_attr {
* out: length of cxt_out */
};
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_test_run_opts() instead")
-LIBBPF_API int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr);
-
-/*
- * bpf_prog_test_run does not check that data_out is large enough. Consider
- * using bpf_prog_test_run_opts instead.
- */
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_test_run_opts() instead")
-LIBBPF_API int bpf_prog_test_run(int prog_fd, int repeat, void *data,
- __u32 size, void *data_out, __u32 *size_out,
- __u32 *retval, __u32 *duration);
LIBBPF_API int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id);
LIBBPF_API int bpf_map_get_next_id(__u32 start_id, __u32 *next_id);
LIBBPF_API int bpf_btf_get_next_id(__u32 start_id, __u32 *next_id);
@@ -442,9 +370,24 @@ LIBBPF_API int bpf_map_get_fd_by_id(__u32 id);
LIBBPF_API int bpf_btf_get_fd_by_id(__u32 id);
LIBBPF_API int bpf_link_get_fd_by_id(__u32 id);
LIBBPF_API int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len);
+
+struct bpf_prog_query_opts {
+ size_t sz; /* size of this struct for forward/backward compatibility */
+ __u32 query_flags;
+ __u32 attach_flags; /* output argument */
+ __u32 *prog_ids;
+ __u32 prog_cnt; /* input+output argument */
+ __u32 *prog_attach_flags;
+};
+#define bpf_prog_query_opts__last_field prog_attach_flags
+
+LIBBPF_API int bpf_prog_query_opts(int target_fd,
+ enum bpf_attach_type type,
+ struct bpf_prog_query_opts *opts);
LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type,
__u32 query_flags, __u32 *attach_flags,
__u32 *prog_ids, __u32 *prog_cnt);
+
LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd);
LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf,
__u32 *buf_len, __u32 *prog_id, __u32 *fd_type,
diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h
index fd48b1ff59ca..496e6a8ee0dc 100644
--- a/tools/lib/bpf/bpf_core_read.h
+++ b/tools/lib/bpf/bpf_core_read.h
@@ -29,6 +29,7 @@ enum bpf_type_id_kind {
enum bpf_type_info_kind {
BPF_TYPE_EXISTS = 0, /* type existence in target kernel */
BPF_TYPE_SIZE = 1, /* type size in target kernel */
+ BPF_TYPE_MATCHES = 2, /* type match in target kernel */
};
/* second argument to __builtin_preserve_enum_value() built-in */
@@ -184,6 +185,16 @@ enum bpf_enum_value_kind {
__builtin_preserve_type_info(*(typeof(type) *)0, BPF_TYPE_EXISTS)
/*
+ * Convenience macro to check that provided named type
+ * (struct/union/enum/typedef) "matches" that in a target kernel.
+ * Returns:
+ * 1, if the type matches in the target kernel's BTF;
+ * 0, if the type does not match any in the target kernel
+ */
+#define bpf_core_type_matches(type) \
+ __builtin_preserve_type_info(*(typeof(type) *)0, BPF_TYPE_MATCHES)
+
+/*
* Convenience macro to get the byte size of a provided named type
* (struct/union/enum/typedef) in a target kernel.
* Returns:
diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h
index fb04eaf367f1..7349b16b8e2f 100644
--- a/tools/lib/bpf/bpf_helpers.h
+++ b/tools/lib/bpf/bpf_helpers.h
@@ -22,12 +22,25 @@
* To allow use of SEC() with externs (e.g., for extern .maps declarations),
* make sure __attribute__((unused)) doesn't trigger compilation warning.
*/
+#if __GNUC__ && !__clang__
+
+/*
+ * Pragma macros are broken on GCC
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400
+ */
+#define SEC(name) __attribute__((section(name), used))
+
+#else
+
#define SEC(name) \
_Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \
__attribute__((section(name), used)) \
_Pragma("GCC diagnostic pop") \
+#endif
+
/* Avoid 'linux/stddef.h' definition of '__always_inline'. */
#undef __always_inline
#define __always_inline inline __attribute__((always_inline))
diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h
index 01ce121c302d..43ca3aff2292 100644
--- a/tools/lib/bpf/bpf_tracing.h
+++ b/tools/lib/bpf/bpf_tracing.h
@@ -2,6 +2,8 @@
#ifndef __BPF_TRACING_H__
#define __BPF_TRACING_H__
+#include <bpf/bpf_helpers.h>
+
/* Scan the ARCH passed in from ARCH env variable (see Makefile) */
#if defined(__TARGET_ARCH_x86)
#define bpf_target_x86
@@ -140,7 +142,7 @@ struct pt_regs___s390 {
#define __PT_RC_REG gprs[2]
#define __PT_SP_REG gprs[15]
#define __PT_IP_REG psw.addr
-#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; })
+#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___s390 *)(x), orig_gpr2)
#elif defined(bpf_target_arm)
@@ -174,7 +176,7 @@ struct pt_regs___arm64 {
#define __PT_RC_REG regs[0]
#define __PT_SP_REG sp
#define __PT_IP_REG pc
-#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; })
+#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x)
#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___arm64 *)(x), orig_x0)
#elif defined(bpf_target_mips)
@@ -233,7 +235,7 @@ struct pt_regs___arm64 {
#define __PT_PARM5_REG a4
#define __PT_RET_REG ra
#define __PT_FP_REG s0
-#define __PT_RC_REG a5
+#define __PT_RC_REG a0
#define __PT_SP_REG sp
#define __PT_IP_REG pc
/* riscv does not select ARCH_HAS_SYSCALL_WRAPPER. */
@@ -493,39 +495,69 @@ typeof(name(0)) name(struct pt_regs *ctx) \
} \
static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args)
+/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */
#define ___bpf_syscall_args0() ctx
-#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs)
-#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs)
-#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs)
-#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs)
-#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs)
+#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs)
+#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs)
+#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs)
+#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs)
+#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs)
#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args)
+/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */
+#define ___bpf_syswrap_args0() ctx
+#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs)
+#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args)
+
/*
- * BPF_KPROBE_SYSCALL is a variant of BPF_KPROBE, which is intended for
+ * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for
* tracing syscall functions, like __x64_sys_close. It hides the underlying
* platform-specific low-level way of getting syscall input arguments from
* struct pt_regs, and provides a familiar typed and named function arguments
* syntax and semantics of accessing syscall input parameters.
*
- * Original struct pt_regs* context is preserved as 'ctx' argument. This might
+ * Original struct pt_regs * context is preserved as 'ctx' argument. This might
* be necessary when using BPF helpers like bpf_perf_event_output().
*
- * This macro relies on BPF CO-RE support.
+ * At the moment BPF_KSYSCALL does not transparently handle all the calling
+ * convention quirks for the following syscalls:
+ *
+ * - mmap(): __ARCH_WANT_SYS_OLD_MMAP.
+ * - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and
+ * CONFIG_CLONE_BACKWARDS3.
+ * - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL.
+ * - compat syscalls.
+ *
+ * This may or may not change in the future. User needs to take extra measures
+ * to handle such quirks explicitly, if necessary.
+ *
+ * This macro relies on BPF CO-RE support and virtual __kconfig externs.
*/
-#define BPF_KPROBE_SYSCALL(name, args...) \
+#define BPF_KSYSCALL(name, args...) \
name(struct pt_regs *ctx); \
+extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \
static __attribute__((always_inline)) typeof(name(0)) \
____##name(struct pt_regs *ctx, ##args); \
typeof(name(0)) name(struct pt_regs *ctx) \
{ \
- struct pt_regs *regs = PT_REGS_SYSCALL_REGS(ctx); \
+ struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \
+ ? (struct pt_regs *)PT_REGS_PARM1(ctx) \
+ : ctx; \
_Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
- return ____##name(___bpf_syscall_args(args)); \
+ if (LINUX_HAS_SYSCALL_WRAPPER) \
+ return ____##name(___bpf_syswrap_args(args)); \
+ else \
+ return ____##name(___bpf_syscall_args(args)); \
_Pragma("GCC diagnostic pop") \
} \
static __attribute__((always_inline)) typeof(name(0)) \
____##name(struct pt_regs *ctx, ##args)
+#define BPF_KPROBE_SYSCALL BPF_KSYSCALL
+
#endif
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index bb1e06eb1eca..2d14f1a52d7a 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -130,7 +130,7 @@ static inline __u64 ptr_to_u64(const void *ptr)
/* Ensure given dynamically allocated memory region pointed to by *data* with
* capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough
- * memory to accomodate *add_cnt* new elements, assuming *cur_cnt* elements
+ * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements
* are already used. At most *max_cnt* elements can be ever allocated.
* If necessary, memory is reallocated and all existing data is copied over,
* new pointer to the memory region is stored at *data, new memory region
@@ -305,6 +305,8 @@ static int btf_type_size(const struct btf_type *t)
return base_size + sizeof(__u32);
case BTF_KIND_ENUM:
return base_size + vlen * sizeof(struct btf_enum);
+ case BTF_KIND_ENUM64:
+ return base_size + vlen * sizeof(struct btf_enum64);
case BTF_KIND_ARRAY:
return base_size + sizeof(struct btf_array);
case BTF_KIND_STRUCT:
@@ -334,6 +336,7 @@ static void btf_bswap_type_base(struct btf_type *t)
static int btf_bswap_type_rest(struct btf_type *t)
{
struct btf_var_secinfo *v;
+ struct btf_enum64 *e64;
struct btf_member *m;
struct btf_array *a;
struct btf_param *p;
@@ -361,6 +364,13 @@ static int btf_bswap_type_rest(struct btf_type *t)
e->val = bswap_32(e->val);
}
return 0;
+ case BTF_KIND_ENUM64:
+ for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) {
+ e64->name_off = bswap_32(e64->name_off);
+ e64->val_lo32 = bswap_32(e64->val_lo32);
+ e64->val_hi32 = bswap_32(e64->val_hi32);
+ }
+ return 0;
case BTF_KIND_ARRAY:
a = btf_array(t);
a->type = bswap_32(a->type);
@@ -438,11 +448,6 @@ static int btf_parse_type_sec(struct btf *btf)
return 0;
}
-__u32 btf__get_nr_types(const struct btf *btf)
-{
- return btf->start_id + btf->nr_types - 1;
-}
-
__u32 btf__type_cnt(const struct btf *btf)
{
return btf->start_id + btf->nr_types;
@@ -472,9 +477,22 @@ const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id)
static int determine_ptr_size(const struct btf *btf)
{
+ static const char * const long_aliases[] = {
+ "long",
+ "long int",
+ "int long",
+ "unsigned long",
+ "long unsigned",
+ "unsigned long int",
+ "unsigned int long",
+ "long unsigned int",
+ "long int unsigned",
+ "int unsigned long",
+ "int long unsigned",
+ };
const struct btf_type *t;
const char *name;
- int i, n;
+ int i, j, n;
if (btf->base_btf && btf->base_btf->ptr_sz > 0)
return btf->base_btf->ptr_sz;
@@ -485,15 +503,16 @@ static int determine_ptr_size(const struct btf *btf)
if (!btf_is_int(t))
continue;
+ if (t->size != 4 && t->size != 8)
+ continue;
+
name = btf__name_by_offset(btf, t->name_off);
if (!name)
continue;
- if (strcmp(name, "long int") == 0 ||
- strcmp(name, "long unsigned int") == 0) {
- if (t->size != 4 && t->size != 8)
- continue;
- return t->size;
+ for (j = 0; j < ARRAY_SIZE(long_aliases); j++) {
+ if (strcmp(name, long_aliases[j]) == 0)
+ return t->size;
}
}
@@ -597,6 +616,7 @@ __s64 btf__resolve_size(const struct btf *btf, __u32 type_id)
case BTF_KIND_STRUCT:
case BTF_KIND_UNION:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
case BTF_KIND_DATASEC:
case BTF_KIND_FLOAT:
size = t->size;
@@ -644,6 +664,7 @@ int btf__align_of(const struct btf *btf, __u32 id)
switch (kind) {
case BTF_KIND_INT:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
case BTF_KIND_FLOAT:
return min(btf_ptr_sz(btf), (size_t)t->size);
case BTF_KIND_PTR:
@@ -1382,92 +1403,6 @@ struct btf *btf__load_from_kernel_by_id(__u32 id)
return btf__load_from_kernel_by_id_split(id, NULL);
}
-int btf__get_from_id(__u32 id, struct btf **btf)
-{
- struct btf *res;
- int err;
-
- *btf = NULL;
- res = btf__load_from_kernel_by_id(id);
- err = libbpf_get_error(res);
-
- if (err)
- return libbpf_err(err);
-
- *btf = res;
- return 0;
-}
-
-int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
- __u32 expected_key_size, __u32 expected_value_size,
- __u32 *key_type_id, __u32 *value_type_id)
-{
- const struct btf_type *container_type;
- const struct btf_member *key, *value;
- const size_t max_name = 256;
- char container_name[max_name];
- __s64 key_size, value_size;
- __s32 container_id;
-
- if (snprintf(container_name, max_name, "____btf_map_%s", map_name) == max_name) {
- pr_warn("map:%s length of '____btf_map_%s' is too long\n",
- map_name, map_name);
- return libbpf_err(-EINVAL);
- }
-
- container_id = btf__find_by_name(btf, container_name);
- if (container_id < 0) {
- pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
- map_name, container_name);
- return libbpf_err(container_id);
- }
-
- container_type = btf__type_by_id(btf, container_id);
- if (!container_type) {
- pr_warn("map:%s cannot find BTF type for container_id:%u\n",
- map_name, container_id);
- return libbpf_err(-EINVAL);
- }
-
- if (!btf_is_struct(container_type) || btf_vlen(container_type) < 2) {
- pr_warn("map:%s container_name:%s is an invalid container struct\n",
- map_name, container_name);
- return libbpf_err(-EINVAL);
- }
-
- key = btf_members(container_type);
- value = key + 1;
-
- key_size = btf__resolve_size(btf, key->type);
- if (key_size < 0) {
- pr_warn("map:%s invalid BTF key_type_size\n", map_name);
- return libbpf_err(key_size);
- }
-
- if (expected_key_size != key_size) {
- pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
- map_name, (__u32)key_size, expected_key_size);
- return libbpf_err(-EINVAL);
- }
-
- value_size = btf__resolve_size(btf, value->type);
- if (value_size < 0) {
- pr_warn("map:%s invalid BTF value_type_size\n", map_name);
- return libbpf_err(value_size);
- }
-
- if (expected_value_size != value_size) {
- pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
- map_name, (__u32)value_size, expected_value_size);
- return libbpf_err(-EINVAL);
- }
-
- *key_type_id = key->type;
- *value_type_id = value->type;
-
- return 0;
-}
-
static void btf_invalidate_raw_data(struct btf *btf)
{
if (btf->raw_data) {
@@ -2115,20 +2050,8 @@ int btf__add_field(struct btf *btf, const char *name, int type_id,
return 0;
}
-/*
- * Append new BTF_KIND_ENUM type with:
- * - *name* - name of the enum, can be NULL or empty for anonymous enums;
- * - *byte_sz* - size of the enum, in bytes.
- *
- * Enum initially has no enum values in it (and corresponds to enum forward
- * declaration). Enumerator values can be added by btf__add_enum_value()
- * immediately after btf__add_enum() succeeds.
- *
- * Returns:
- * - >0, type ID of newly added BTF type;
- * - <0, on error.
- */
-int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
+static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz,
+ bool is_signed, __u8 kind)
{
struct btf_type *t;
int sz, name_off = 0;
@@ -2153,13 +2076,35 @@ int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
/* start out with vlen=0; it will be adjusted when adding enum values */
t->name_off = name_off;
- t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);
+ t->info = btf_type_info(kind, 0, is_signed);
t->size = byte_sz;
return btf_commit_type(btf, sz);
}
/*
+ * Append new BTF_KIND_ENUM type with:
+ * - *name* - name of the enum, can be NULL or empty for anonymous enums;
+ * - *byte_sz* - size of the enum, in bytes.
+ *
+ * Enum initially has no enum values in it (and corresponds to enum forward
+ * declaration). Enumerator values can be added by btf__add_enum_value()
+ * immediately after btf__add_enum() succeeds.
+ *
+ * Returns:
+ * - >0, type ID of newly added BTF type;
+ * - <0, on error.
+ */
+int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz)
+{
+ /*
+ * set the signedness to be unsigned, it will change to signed
+ * if any later enumerator is negative.
+ */
+ return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM);
+}
+
+/*
* Append new enum value for the current ENUM type with:
* - *name* - name of the enumerator value, can't be NULL or empty;
* - *value* - integer value corresponding to enum value *name*;
@@ -2206,6 +2151,82 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value)
t = btf_last_type(btf);
btf_type_inc_vlen(t);
+ /* if negative value, set signedness to signed */
+ if (value < 0)
+ t->info = btf_type_info(btf_kind(t), btf_vlen(t), true);
+
+ btf->hdr->type_len += sz;
+ btf->hdr->str_off += sz;
+ return 0;
+}
+
+/*
+ * Append new BTF_KIND_ENUM64 type with:
+ * - *name* - name of the enum, can be NULL or empty for anonymous enums;
+ * - *byte_sz* - size of the enum, in bytes.
+ * - *is_signed* - whether the enum values are signed or not;
+ *
+ * Enum initially has no enum values in it (and corresponds to enum forward
+ * declaration). Enumerator values can be added by btf__add_enum64_value()
+ * immediately after btf__add_enum64() succeeds.
+ *
+ * Returns:
+ * - >0, type ID of newly added BTF type;
+ * - <0, on error.
+ */
+int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz,
+ bool is_signed)
+{
+ return btf_add_enum_common(btf, name, byte_sz, is_signed,
+ BTF_KIND_ENUM64);
+}
+
+/*
+ * Append new enum value for the current ENUM64 type with:
+ * - *name* - name of the enumerator value, can't be NULL or empty;
+ * - *value* - integer value corresponding to enum value *name*;
+ * Returns:
+ * - 0, on success;
+ * - <0, on error.
+ */
+int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value)
+{
+ struct btf_enum64 *v;
+ struct btf_type *t;
+ int sz, name_off;
+
+ /* last type should be BTF_KIND_ENUM64 */
+ if (btf->nr_types == 0)
+ return libbpf_err(-EINVAL);
+ t = btf_last_type(btf);
+ if (!btf_is_enum64(t))
+ return libbpf_err(-EINVAL);
+
+ /* non-empty name */
+ if (!name || !name[0])
+ return libbpf_err(-EINVAL);
+
+ /* decompose and invalidate raw data */
+ if (btf_ensure_modifiable(btf))
+ return libbpf_err(-ENOMEM);
+
+ sz = sizeof(struct btf_enum64);
+ v = btf_add_type_mem(btf, sz);
+ if (!v)
+ return libbpf_err(-ENOMEM);
+
+ name_off = btf__add_str(btf, name);
+ if (name_off < 0)
+ return name_off;
+
+ v->name_off = name_off;
+ v->val_lo32 = (__u32)value;
+ v->val_hi32 = value >> 32;
+
+ /* update parent type's vlen */
+ t = btf_last_type(btf);
+ btf_type_inc_vlen(t);
+
btf->hdr->type_len += sz;
btf->hdr->str_off += sz;
return 0;
@@ -2853,81 +2874,6 @@ const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size)
return btf_ext->data;
}
-static int btf_ext_reloc_info(const struct btf *btf,
- const struct btf_ext_info *ext_info,
- const char *sec_name, __u32 insns_cnt,
- void **info, __u32 *cnt)
-{
- __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec);
- __u32 i, record_size, existing_len, records_len;
- struct btf_ext_info_sec *sinfo;
- const char *info_sec_name;
- __u64 remain_len;
- void *data;
-
- record_size = ext_info->rec_size;
- sinfo = ext_info->info;
- remain_len = ext_info->len;
- while (remain_len > 0) {
- records_len = sinfo->num_info * record_size;
- info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off);
- if (strcmp(info_sec_name, sec_name)) {
- remain_len -= sec_hdrlen + records_len;
- sinfo = (void *)sinfo + sec_hdrlen + records_len;
- continue;
- }
-
- existing_len = (*cnt) * record_size;
- data = realloc(*info, existing_len + records_len);
- if (!data)
- return libbpf_err(-ENOMEM);
-
- memcpy(data + existing_len, sinfo->data, records_len);
- /* adjust insn_off only, the rest data will be passed
- * to the kernel.
- */
- for (i = 0; i < sinfo->num_info; i++) {
- __u32 *insn_off;
-
- insn_off = data + existing_len + (i * record_size);
- *insn_off = *insn_off / sizeof(struct bpf_insn) + insns_cnt;
- }
- *info = data;
- *cnt += sinfo->num_info;
- return 0;
- }
-
- return libbpf_err(-ENOENT);
-}
-
-int btf_ext__reloc_func_info(const struct btf *btf,
- const struct btf_ext *btf_ext,
- const char *sec_name, __u32 insns_cnt,
- void **func_info, __u32 *cnt)
-{
- return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name,
- insns_cnt, func_info, cnt);
-}
-
-int btf_ext__reloc_line_info(const struct btf *btf,
- const struct btf_ext *btf_ext,
- const char *sec_name, __u32 insns_cnt,
- void **line_info, __u32 *cnt)
-{
- return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name,
- insns_cnt, line_info, cnt);
-}
-
-__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext)
-{
- return btf_ext->func_info.rec_size;
-}
-
-__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext)
-{
- return btf_ext->line_info.rec_size;
-}
-
struct btf_dedup;
static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts);
@@ -3077,9 +3023,7 @@ static int btf_dedup_remap_types(struct btf_dedup *d);
* deduplicating structs/unions is described in greater details in comments for
* `btf_dedup_is_equiv` function.
*/
-
-DEFAULT_VERSION(btf__dedup_v0_6_0, btf__dedup, LIBBPF_0.6.0)
-int btf__dedup_v0_6_0(struct btf *btf, const struct btf_dedup_opts *opts)
+int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts)
{
struct btf_dedup *d;
int err;
@@ -3139,19 +3083,6 @@ done:
return libbpf_err(err);
}
-COMPAT_VERSION(btf__dedup_deprecated, btf__dedup, LIBBPF_0.0.2)
-int btf__dedup_deprecated(struct btf *btf, struct btf_ext *btf_ext, const void *unused_opts)
-{
- LIBBPF_OPTS(btf_dedup_opts, opts, .btf_ext = btf_ext);
-
- if (unused_opts) {
- pr_warn("please use new version of btf__dedup() that supports options\n");
- return libbpf_err(-ENOTSUP);
- }
-
- return btf__dedup(btf, &opts);
-}
-
#define BTF_UNPROCESSED_ID ((__u32)-1)
#define BTF_IN_PROGRESS_ID ((__u32)-2)
@@ -3470,7 +3401,7 @@ static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2)
return info1 == info2;
}
-/* Calculate type signature hash of ENUM. */
+/* Calculate type signature hash of ENUM/ENUM64. */
static long btf_hash_enum(struct btf_type *t)
{
long h;
@@ -3504,9 +3435,31 @@ static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2)
return true;
}
+static bool btf_equal_enum64(struct btf_type *t1, struct btf_type *t2)
+{
+ const struct btf_enum64 *m1, *m2;
+ __u16 vlen;
+ int i;
+
+ if (!btf_equal_common(t1, t2))
+ return false;
+
+ vlen = btf_vlen(t1);
+ m1 = btf_enum64(t1);
+ m2 = btf_enum64(t2);
+ for (i = 0; i < vlen; i++) {
+ if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 ||
+ m1->val_hi32 != m2->val_hi32)
+ return false;
+ m1++;
+ m2++;
+ }
+ return true;
+}
+
static inline bool btf_is_enum_fwd(struct btf_type *t)
{
- return btf_is_enum(t) && btf_vlen(t) == 0;
+ return btf_is_any_enum(t) && btf_vlen(t) == 0;
}
static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2)
@@ -3519,6 +3472,17 @@ static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2)
t1->size == t2->size;
}
+static bool btf_compat_enum64(struct btf_type *t1, struct btf_type *t2)
+{
+ if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2))
+ return btf_equal_enum64(t1, t2);
+
+ /* ignore vlen when comparing */
+ return t1->name_off == t2->name_off &&
+ (t1->info & ~0xffff) == (t2->info & ~0xffff) &&
+ t1->size == t2->size;
+}
+
/*
* Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs,
* as referenced type IDs equivalence is established separately during type
@@ -3731,6 +3695,7 @@ static int btf_dedup_prep(struct btf_dedup *d)
h = btf_hash_int_decl_tag(t);
break;
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
h = btf_hash_enum(t);
break;
case BTF_KIND_STRUCT:
@@ -3820,6 +3785,27 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)
}
break;
+ case BTF_KIND_ENUM64:
+ h = btf_hash_enum(t);
+ for_each_dedup_cand(d, hash_entry, h) {
+ cand_id = (__u32)(long)hash_entry->value;
+ cand = btf_type_by_id(d->btf, cand_id);
+ if (btf_equal_enum64(t, cand)) {
+ new_id = cand_id;
+ break;
+ }
+ if (btf_compat_enum64(t, cand)) {
+ if (btf_is_enum_fwd(t)) {
+ /* resolve fwd to full enum */
+ new_id = cand_id;
+ break;
+ }
+ /* resolve canonical enum fwd to full enum */
+ d->map[cand_id] = type_id;
+ }
+ }
+ break;
+
case BTF_KIND_FWD:
case BTF_KIND_FLOAT:
h = btf_hash_common(t);
@@ -4115,6 +4101,9 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id,
case BTF_KIND_ENUM:
return btf_compat_enum(cand_type, canon_type);
+ case BTF_KIND_ENUM64:
+ return btf_compat_enum64(cand_type, canon_type);
+
case BTF_KIND_FWD:
case BTF_KIND_FLOAT:
return btf_equal_common(cand_type, canon_type);
@@ -4717,6 +4706,7 @@ int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ct
case BTF_KIND_INT:
case BTF_KIND_FLOAT:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
return 0;
case BTF_KIND_FWD:
@@ -4811,6 +4801,16 @@ int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ct
}
break;
}
+ case BTF_KIND_ENUM64: {
+ struct btf_enum64 *m = btf_enum64(t);
+
+ for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
+ err = visit(&m->name_off, ctx);
+ if (err)
+ return err;
+ }
+ break;
+ }
case BTF_KIND_FUNC_PROTO: {
struct btf_param *m = btf_params(t);
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index 951ac7475794..583760df83b4 100644
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -120,20 +120,12 @@ LIBBPF_API struct btf *libbpf_find_kernel_btf(void);
LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id);
LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf);
-LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_from_kernel_by_id instead")
-LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
-LIBBPF_DEPRECATED_SINCE(0, 6, "intended for internal libbpf use only")
-LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);
-LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_into_kernel instead")
-LIBBPF_API int btf__load(struct btf *btf);
LIBBPF_API int btf__load_into_kernel(struct btf *btf);
LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
const char *type_name);
LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf,
const char *type_name, __u32 kind);
-LIBBPF_DEPRECATED_SINCE(0, 7, "use btf__type_cnt() instead; note that btf__get_nr_types() == btf__type_cnt() - 1")
-LIBBPF_API __u32 btf__get_nr_types(const struct btf *btf);
LIBBPF_API __u32 btf__type_cnt(const struct btf *btf);
LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf);
LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf,
@@ -150,29 +142,10 @@ LIBBPF_API void btf__set_fd(struct btf *btf, int fd);
LIBBPF_API const void *btf__raw_data(const struct btf *btf, __u32 *size);
LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset);
LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset);
-LIBBPF_DEPRECATED_SINCE(0, 7, "this API is not necessary when BTF-defined maps are used")
-LIBBPF_API int btf__get_map_kv_tids(const struct btf *btf, const char *map_name,
- __u32 expected_key_size,
- __u32 expected_value_size,
- __u32 *key_type_id, __u32 *value_type_id);
LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size);
LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext);
LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size);
-LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_func_info was never meant as a public API and has wrong assumptions embedded in it; it will be removed in the future libbpf versions")
-int btf_ext__reloc_func_info(const struct btf *btf,
- const struct btf_ext *btf_ext,
- const char *sec_name, __u32 insns_cnt,
- void **func_info, __u32 *cnt);
-LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_line_info was never meant as a public API and has wrong assumptions embedded in it; it will be removed in the future libbpf versions")
-int btf_ext__reloc_line_info(const struct btf *btf,
- const struct btf_ext *btf_ext,
- const char *sec_name, __u32 insns_cnt,
- void **line_info, __u32 *cnt);
-LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_func_info is deprecated; write custom func_info parsing to fetch rec_size")
-__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext);
-LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_line_info is deprecated; write custom line_info parsing to fetch rec_size")
-__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext);
LIBBPF_API int btf__find_str(struct btf *btf, const char *s);
LIBBPF_API int btf__add_str(struct btf *btf, const char *s);
@@ -215,6 +188,8 @@ LIBBPF_API int btf__add_field(struct btf *btf, const char *name, int field_type_
/* enum construction APIs */
LIBBPF_API int btf__add_enum(struct btf *btf, const char *name, __u32 bytes_sz);
LIBBPF_API int btf__add_enum_value(struct btf *btf, const char *name, __s64 value);
+LIBBPF_API int btf__add_enum64(struct btf *btf, const char *name, __u32 bytes_sz, bool is_signed);
+LIBBPF_API int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value);
enum btf_fwd_kind {
BTF_FWD_STRUCT = 0,
@@ -257,22 +232,12 @@ struct btf_dedup_opts {
LIBBPF_API int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts);
-LIBBPF_API int btf__dedup_v0_6_0(struct btf *btf, const struct btf_dedup_opts *opts);
-
-LIBBPF_DEPRECATED_SINCE(0, 7, "use btf__dedup() instead")
-LIBBPF_API int btf__dedup_deprecated(struct btf *btf, struct btf_ext *btf_ext, const void *opts);
-#define btf__dedup(...) ___libbpf_overload(___btf_dedup, __VA_ARGS__)
-#define ___btf_dedup3(btf, btf_ext, opts) btf__dedup_deprecated(btf, btf_ext, opts)
-#define ___btf_dedup2(btf, opts) btf__dedup(btf, opts)
-
struct btf_dump;
struct btf_dump_opts {
- union {
- size_t sz;
- void *ctx; /* DEPRECATED: will be gone in v1.0 */
- };
+ size_t sz;
};
+#define btf_dump_opts__last_field sz
typedef void (*btf_dump_printf_fn_t)(void *ctx, const char *fmt, va_list args);
@@ -281,51 +246,6 @@ LIBBPF_API struct btf_dump *btf_dump__new(const struct btf *btf,
void *ctx,
const struct btf_dump_opts *opts);
-LIBBPF_API struct btf_dump *btf_dump__new_v0_6_0(const struct btf *btf,
- btf_dump_printf_fn_t printf_fn,
- void *ctx,
- const struct btf_dump_opts *opts);
-
-LIBBPF_API struct btf_dump *btf_dump__new_deprecated(const struct btf *btf,
- const struct btf_ext *btf_ext,
- const struct btf_dump_opts *opts,
- btf_dump_printf_fn_t printf_fn);
-
-/* Choose either btf_dump__new() or btf_dump__new_deprecated() based on the
- * type of 4th argument. If it's btf_dump's print callback, use deprecated
- * API; otherwise, choose the new btf_dump__new(). ___libbpf_override()
- * doesn't work here because both variants have 4 input arguments.
- *
- * (void *) casts are necessary to avoid compilation warnings about type
- * mismatches, because even though __builtin_choose_expr() only ever evaluates
- * one side the other side still has to satisfy type constraints (this is
- * compiler implementation limitation which might be lifted eventually,
- * according to the documentation). So passing struct btf_ext in place of
- * btf_dump_printf_fn_t would be generating compilation warning. Casting to
- * void * avoids this issue.
- *
- * Also, two type compatibility checks for a function and function pointer are
- * required because passing function reference into btf_dump__new() as
- * btf_dump__new(..., my_callback, ...) and as btf_dump__new(...,
- * &my_callback, ...) (not explicit ampersand in the latter case) actually
- * differs as far as __builtin_types_compatible_p() is concerned. Thus two
- * checks are combined to detect callback argument.
- *
- * The rest works just like in case of ___libbpf_override() usage with symbol
- * versioning.
- *
- * C++ compilers don't support __builtin_types_compatible_p(), so at least
- * don't screw up compilation for them and let C++ users pick btf_dump__new
- * vs btf_dump__new_deprecated explicitly.
- */
-#ifndef __cplusplus
-#define btf_dump__new(a1, a2, a3, a4) __builtin_choose_expr( \
- __builtin_types_compatible_p(typeof(a4), btf_dump_printf_fn_t) || \
- __builtin_types_compatible_p(typeof(a4), void(void *, const char *, va_list)), \
- btf_dump__new_deprecated((void *)a1, (void *)a2, (void *)a3, (void *)a4), \
- btf_dump__new((void *)a1, (void *)a2, (void *)a3, (void *)a4))
-#endif
-
LIBBPF_API void btf_dump__free(struct btf_dump *d);
LIBBPF_API int btf_dump__dump_type(struct btf_dump *d, __u32 id);
@@ -393,9 +313,10 @@ btf_dump__dump_type_data(struct btf_dump *d, __u32 id,
#ifndef BTF_KIND_FLOAT
#define BTF_KIND_FLOAT 16 /* Floating point */
#endif
-/* The kernel header switched to enums, so these two were never #defined */
+/* The kernel header switched to enums, so the following were never #defined */
#define BTF_KIND_DECL_TAG 17 /* Decl Tag */
#define BTF_KIND_TYPE_TAG 18 /* Type Tag */
+#define BTF_KIND_ENUM64 19 /* Enum for up-to 64bit values */
static inline __u16 btf_kind(const struct btf_type *t)
{
@@ -454,6 +375,11 @@ static inline bool btf_is_enum(const struct btf_type *t)
return btf_kind(t) == BTF_KIND_ENUM;
}
+static inline bool btf_is_enum64(const struct btf_type *t)
+{
+ return btf_kind(t) == BTF_KIND_ENUM64;
+}
+
static inline bool btf_is_fwd(const struct btf_type *t)
{
return btf_kind(t) == BTF_KIND_FWD;
@@ -524,6 +450,18 @@ static inline bool btf_is_type_tag(const struct btf_type *t)
return btf_kind(t) == BTF_KIND_TYPE_TAG;
}
+static inline bool btf_is_any_enum(const struct btf_type *t)
+{
+ return btf_is_enum(t) || btf_is_enum64(t);
+}
+
+static inline bool btf_kind_core_compat(const struct btf_type *t1,
+ const struct btf_type *t2)
+{
+ return btf_kind(t1) == btf_kind(t2) ||
+ (btf_is_any_enum(t1) && btf_is_any_enum(t2));
+}
+
static inline __u8 btf_int_encoding(const struct btf_type *t)
{
return BTF_INT_ENCODING(*(__u32 *)(t + 1));
@@ -549,6 +487,16 @@ static inline struct btf_enum *btf_enum(const struct btf_type *t)
return (struct btf_enum *)(t + 1);
}
+static inline struct btf_enum64 *btf_enum64(const struct btf_type *t)
+{
+ return (struct btf_enum64 *)(t + 1);
+}
+
+static inline __u64 btf_enum64_value(const struct btf_enum64 *e)
+{
+ return ((__u64)e->val_hi32 << 32) | e->val_lo32;
+}
+
static inline struct btf_member *btf_members(const struct btf_type *t)
{
return (struct btf_member *)(t + 1);
diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c
index 6b1bc1f43728..627edb5bb6de 100644
--- a/tools/lib/bpf/btf_dump.c
+++ b/tools/lib/bpf/btf_dump.c
@@ -144,15 +144,17 @@ static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...)
static int btf_dump_mark_referenced(struct btf_dump *d);
static int btf_dump_resize(struct btf_dump *d);
-DEFAULT_VERSION(btf_dump__new_v0_6_0, btf_dump__new, LIBBPF_0.6.0)
-struct btf_dump *btf_dump__new_v0_6_0(const struct btf *btf,
- btf_dump_printf_fn_t printf_fn,
- void *ctx,
- const struct btf_dump_opts *opts)
+struct btf_dump *btf_dump__new(const struct btf *btf,
+ btf_dump_printf_fn_t printf_fn,
+ void *ctx,
+ const struct btf_dump_opts *opts)
{
struct btf_dump *d;
int err;
+ if (!OPTS_VALID(opts, btf_dump_opts))
+ return libbpf_err_ptr(-EINVAL);
+
if (!printf_fn)
return libbpf_err_ptr(-EINVAL);
@@ -188,17 +190,6 @@ err:
return libbpf_err_ptr(err);
}
-COMPAT_VERSION(btf_dump__new_deprecated, btf_dump__new, LIBBPF_0.0.4)
-struct btf_dump *btf_dump__new_deprecated(const struct btf *btf,
- const struct btf_ext *btf_ext,
- const struct btf_dump_opts *opts,
- btf_dump_printf_fn_t printf_fn)
-{
- if (!printf_fn)
- return libbpf_err_ptr(-EINVAL);
- return btf_dump__new_v0_6_0(btf, printf_fn, opts ? opts->ctx : NULL, opts);
-}
-
static int btf_dump_resize(struct btf_dump *d)
{
int err, last_id = btf__type_cnt(d->btf) - 1;
@@ -318,6 +309,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d)
switch (btf_kind(t)) {
case BTF_KIND_INT:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
case BTF_KIND_FWD:
case BTF_KIND_FLOAT:
break;
@@ -538,6 +530,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr)
return 1;
}
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
case BTF_KIND_FWD:
/*
* non-anonymous or non-referenced enums are top-level
@@ -739,6 +732,7 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id)
tstate->emit_state = EMITTED;
break;
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
if (top_level_def) {
btf_dump_emit_enum_def(d, id, t, 0);
btf_dump_printf(d, ";\n\n");
@@ -989,38 +983,81 @@ static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id,
btf_dump_printf(d, "enum %s", btf_dump_type_name(d, id));
}
-static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id,
- const struct btf_type *t,
- int lvl)
+static void btf_dump_emit_enum32_val(struct btf_dump *d,
+ const struct btf_type *t,
+ int lvl, __u16 vlen)
{
const struct btf_enum *v = btf_enum(t);
- __u16 vlen = btf_vlen(t);
+ bool is_signed = btf_kflag(t);
+ const char *fmt_str;
const char *name;
size_t dup_cnt;
int i;
+ for (i = 0; i < vlen; i++, v++) {
+ name = btf_name_of(d, v->name_off);
+ /* enumerators share namespace with typedef idents */
+ dup_cnt = btf_dump_name_dups(d, d->ident_names, name);
+ if (dup_cnt > 1) {
+ fmt_str = is_signed ? "\n%s%s___%zd = %d," : "\n%s%s___%zd = %u,";
+ btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, dup_cnt, v->val);
+ } else {
+ fmt_str = is_signed ? "\n%s%s = %d," : "\n%s%s = %u,";
+ btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, v->val);
+ }
+ }
+}
+
+static void btf_dump_emit_enum64_val(struct btf_dump *d,
+ const struct btf_type *t,
+ int lvl, __u16 vlen)
+{
+ const struct btf_enum64 *v = btf_enum64(t);
+ bool is_signed = btf_kflag(t);
+ const char *fmt_str;
+ const char *name;
+ size_t dup_cnt;
+ __u64 val;
+ int i;
+
+ for (i = 0; i < vlen; i++, v++) {
+ name = btf_name_of(d, v->name_off);
+ dup_cnt = btf_dump_name_dups(d, d->ident_names, name);
+ val = btf_enum64_value(v);
+ if (dup_cnt > 1) {
+ fmt_str = is_signed ? "\n%s%s___%zd = %lldLL,"
+ : "\n%s%s___%zd = %lluULL,";
+ btf_dump_printf(d, fmt_str,
+ pfx(lvl + 1), name, dup_cnt,
+ (unsigned long long)val);
+ } else {
+ fmt_str = is_signed ? "\n%s%s = %lldLL,"
+ : "\n%s%s = %lluULL,";
+ btf_dump_printf(d, fmt_str,
+ pfx(lvl + 1), name,
+ (unsigned long long)val);
+ }
+ }
+}
+static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id,
+ const struct btf_type *t,
+ int lvl)
+{
+ __u16 vlen = btf_vlen(t);
+
btf_dump_printf(d, "enum%s%s",
t->name_off ? " " : "",
btf_dump_type_name(d, id));
- if (vlen) {
- btf_dump_printf(d, " {");
- for (i = 0; i < vlen; i++, v++) {
- name = btf_name_of(d, v->name_off);
- /* enumerators share namespace with typedef idents */
- dup_cnt = btf_dump_name_dups(d, d->ident_names, name);
- if (dup_cnt > 1) {
- btf_dump_printf(d, "\n%s%s___%zu = %u,",
- pfx(lvl + 1), name, dup_cnt,
- (__u32)v->val);
- } else {
- btf_dump_printf(d, "\n%s%s = %u,",
- pfx(lvl + 1), name,
- (__u32)v->val);
- }
- }
- btf_dump_printf(d, "\n%s}", pfx(lvl));
- }
+ if (!vlen)
+ return;
+
+ btf_dump_printf(d, " {");
+ if (btf_is_enum(t))
+ btf_dump_emit_enum32_val(d, t, lvl, vlen);
+ else
+ btf_dump_emit_enum64_val(d, t, lvl, vlen);
+ btf_dump_printf(d, "\n%s}", pfx(lvl));
}
static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id,
@@ -1178,6 +1215,7 @@ skip_mod:
break;
case BTF_KIND_INT:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
case BTF_KIND_FWD:
case BTF_KIND_STRUCT:
case BTF_KIND_UNION:
@@ -1312,6 +1350,7 @@ static void btf_dump_emit_type_chain(struct btf_dump *d,
btf_dump_emit_struct_fwd(d, id, t);
break;
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
btf_dump_emit_mods(d, decls);
/* inline anonymous enum */
if (t->name_off == 0 && !d->skip_anon_defs)
@@ -1988,7 +2027,8 @@ static int btf_dump_get_enum_value(struct btf_dump *d,
__u32 id,
__s64 *value)
{
- /* handle unaligned enum value */
+ bool is_signed = btf_kflag(t);
+
if (!ptr_is_aligned(d->btf, id, data)) {
__u64 val;
int err;
@@ -2005,13 +2045,13 @@ static int btf_dump_get_enum_value(struct btf_dump *d,
*value = *(__s64 *)data;
return 0;
case 4:
- *value = *(__s32 *)data;
+ *value = is_signed ? (__s64)*(__s32 *)data : *(__u32 *)data;
return 0;
case 2:
- *value = *(__s16 *)data;
+ *value = is_signed ? *(__s16 *)data : *(__u16 *)data;
return 0;
case 1:
- *value = *(__s8 *)data;
+ *value = is_signed ? *(__s8 *)data : *(__u8 *)data;
return 0;
default:
pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id);
@@ -2024,7 +2064,7 @@ static int btf_dump_enum_data(struct btf_dump *d,
__u32 id,
const void *data)
{
- const struct btf_enum *e;
+ bool is_signed;
__s64 value;
int i, err;
@@ -2032,14 +2072,31 @@ static int btf_dump_enum_data(struct btf_dump *d,
if (err)
return err;
- for (i = 0, e = btf_enum(t); i < btf_vlen(t); i++, e++) {
- if (value != e->val)
- continue;
- btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off));
- return 0;
- }
+ is_signed = btf_kflag(t);
+ if (btf_is_enum(t)) {
+ const struct btf_enum *e;
+
+ for (i = 0, e = btf_enum(t); i < btf_vlen(t); i++, e++) {
+ if (value != e->val)
+ continue;
+ btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off));
+ return 0;
+ }
- btf_dump_type_values(d, "%d", value);
+ btf_dump_type_values(d, is_signed ? "%d" : "%u", value);
+ } else {
+ const struct btf_enum64 *e;
+
+ for (i = 0, e = btf_enum64(t); i < btf_vlen(t); i++, e++) {
+ if (value != btf_enum64_value(e))
+ continue;
+ btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off));
+ return 0;
+ }
+
+ btf_dump_type_values(d, is_signed ? "%lldLL" : "%lluULL",
+ (unsigned long long)value);
+ }
return 0;
}
@@ -2099,6 +2156,7 @@ static int btf_dump_type_data_check_overflow(struct btf_dump *d,
case BTF_KIND_FLOAT:
case BTF_KIND_PTR:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
if (data + bits_offset / 8 + size > d->typed_dump->data_end)
return -E2BIG;
break;
@@ -2203,6 +2261,7 @@ static int btf_dump_type_data_check_zero(struct btf_dump *d,
return -ENODATA;
}
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
err = btf_dump_get_enum_value(d, t, data, id, &value);
if (err)
return err;
@@ -2275,6 +2334,7 @@ static int btf_dump_dump_type_data(struct btf_dump *d,
err = btf_dump_struct_data(d, t, id, data);
break;
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
/* handle bitfield and int enum values */
if (bit_sz) {
__u64 print_num;
diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c
index 927745b08014..23f5c46708f8 100644
--- a/tools/lib/bpf/gen_loader.c
+++ b/tools/lib/bpf/gen_loader.c
@@ -533,7 +533,7 @@ void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name,
gen->attach_kind = kind;
ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s",
prefix, attach_name);
- if (ret == sizeof(gen->attach_target))
+ if (ret >= sizeof(gen->attach_target))
gen->error = -ENOSPC;
}
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index e89cc9c885b3..50d41815f431 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -31,7 +31,6 @@
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/filter.h>
-#include <linux/list.h>
#include <linux/limits.h>
#include <linux/perf_event.h>
#include <linux/ring_buffer.h>
@@ -72,6 +71,135 @@
static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
+static const char * const attach_type_name[] = {
+ [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress",
+ [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress",
+ [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create",
+ [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release",
+ [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops",
+ [BPF_CGROUP_DEVICE] = "cgroup_device",
+ [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind",
+ [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind",
+ [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect",
+ [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect",
+ [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind",
+ [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind",
+ [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername",
+ [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername",
+ [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname",
+ [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname",
+ [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg",
+ [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg",
+ [BPF_CGROUP_SYSCTL] = "cgroup_sysctl",
+ [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg",
+ [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg",
+ [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt",
+ [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt",
+ [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
+ [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
+ [BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
+ [BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
+ [BPF_LIRC_MODE2] = "lirc_mode2",
+ [BPF_FLOW_DISSECTOR] = "flow_dissector",
+ [BPF_TRACE_RAW_TP] = "trace_raw_tp",
+ [BPF_TRACE_FENTRY] = "trace_fentry",
+ [BPF_TRACE_FEXIT] = "trace_fexit",
+ [BPF_MODIFY_RETURN] = "modify_return",
+ [BPF_LSM_MAC] = "lsm_mac",
+ [BPF_LSM_CGROUP] = "lsm_cgroup",
+ [BPF_SK_LOOKUP] = "sk_lookup",
+ [BPF_TRACE_ITER] = "trace_iter",
+ [BPF_XDP_DEVMAP] = "xdp_devmap",
+ [BPF_XDP_CPUMAP] = "xdp_cpumap",
+ [BPF_XDP] = "xdp",
+ [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select",
+ [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate",
+ [BPF_PERF_EVENT] = "perf_event",
+ [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi",
+};
+
+static const char * const link_type_name[] = {
+ [BPF_LINK_TYPE_UNSPEC] = "unspec",
+ [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
+ [BPF_LINK_TYPE_TRACING] = "tracing",
+ [BPF_LINK_TYPE_CGROUP] = "cgroup",
+ [BPF_LINK_TYPE_ITER] = "iter",
+ [BPF_LINK_TYPE_NETNS] = "netns",
+ [BPF_LINK_TYPE_XDP] = "xdp",
+ [BPF_LINK_TYPE_PERF_EVENT] = "perf_event",
+ [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
+ [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
+};
+
+static const char * const map_type_name[] = {
+ [BPF_MAP_TYPE_UNSPEC] = "unspec",
+ [BPF_MAP_TYPE_HASH] = "hash",
+ [BPF_MAP_TYPE_ARRAY] = "array",
+ [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array",
+ [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array",
+ [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash",
+ [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array",
+ [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace",
+ [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array",
+ [BPF_MAP_TYPE_LRU_HASH] = "lru_hash",
+ [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash",
+ [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie",
+ [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps",
+ [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps",
+ [BPF_MAP_TYPE_DEVMAP] = "devmap",
+ [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash",
+ [BPF_MAP_TYPE_SOCKMAP] = "sockmap",
+ [BPF_MAP_TYPE_CPUMAP] = "cpumap",
+ [BPF_MAP_TYPE_XSKMAP] = "xskmap",
+ [BPF_MAP_TYPE_SOCKHASH] = "sockhash",
+ [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage",
+ [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray",
+ [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage",
+ [BPF_MAP_TYPE_QUEUE] = "queue",
+ [BPF_MAP_TYPE_STACK] = "stack",
+ [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage",
+ [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops",
+ [BPF_MAP_TYPE_RINGBUF] = "ringbuf",
+ [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage",
+ [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage",
+ [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter",
+};
+
+static const char * const prog_type_name[] = {
+ [BPF_PROG_TYPE_UNSPEC] = "unspec",
+ [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
+ [BPF_PROG_TYPE_KPROBE] = "kprobe",
+ [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
+ [BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
+ [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
+ [BPF_PROG_TYPE_XDP] = "xdp",
+ [BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
+ [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
+ [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
+ [BPF_PROG_TYPE_LWT_IN] = "lwt_in",
+ [BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
+ [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
+ [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
+ [BPF_PROG_TYPE_SK_SKB] = "sk_skb",
+ [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
+ [BPF_PROG_TYPE_SK_MSG] = "sk_msg",
+ [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
+ [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
+ [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local",
+ [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
+ [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
+ [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
+ [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",
+ [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
+ [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt",
+ [BPF_PROG_TYPE_TRACING] = "tracing",
+ [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops",
+ [BPF_PROG_TYPE_EXT] = "ext",
+ [BPF_PROG_TYPE_LSM] = "lsm",
+ [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup",
+ [BPF_PROG_TYPE_SYSCALL] = "syscall",
+};
+
static int __base_pr(enum libbpf_print_level level, const char *format,
va_list args)
{
@@ -151,12 +279,9 @@ static inline __u64 ptr_to_u64(const void *ptr)
return (__u64) (unsigned long) ptr;
}
-/* this goes away in libbpf 1.0 */
-enum libbpf_strict_mode libbpf_mode = LIBBPF_STRICT_NONE;
-
int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
{
- libbpf_mode = mode;
+ /* as of v1.0 libbpf_set_strict_mode() is a no-op */
return 0;
}
@@ -219,12 +344,8 @@ enum sec_def_flags {
SEC_ATTACH_BTF = 4,
/* BPF program type allows sleeping/blocking in kernel */
SEC_SLEEPABLE = 8,
- /* allow non-strict prefix matching */
- SEC_SLOPPY_PFX = 16,
/* BPF program support non-linear XDP buffer */
- SEC_XDP_FRAGS = 32,
- /* deprecated sec definitions not supposed to be used */
- SEC_DEPRECATED = 64,
+ SEC_XDP_FRAGS = 16,
};
struct bpf_sec_def {
@@ -244,9 +365,10 @@ struct bpf_sec_def {
* linux/filter.h.
*/
struct bpf_program {
- const struct bpf_sec_def *sec_def;
+ char *name;
char *sec_name;
size_t sec_idx;
+ const struct bpf_sec_def *sec_def;
/* this program's instruction offset (in number of instructions)
* within its containing ELF section
*/
@@ -266,12 +388,6 @@ struct bpf_program {
*/
size_t sub_insn_off;
- char *name;
- /* name with / replaced by _; makes recursive pinning
- * in bpf_object__pin_programs easier
- */
- char *pin_name;
-
/* instructions that belong to BPF program; insns[0] is located at
* sec_insn_off instruction within its ELF section in ELF file, so
* when mapping ELF file instruction index to the local instruction,
@@ -292,24 +408,19 @@ struct bpf_program {
size_t log_size;
__u32 log_level;
- struct {
- int nr;
- int *fds;
- } instances;
- bpf_program_prep_t preprocessor;
-
struct bpf_object *obj;
- void *priv;
- bpf_program_clear_priv_t clear_priv;
+ int fd;
bool autoload;
bool mark_btf_static;
enum bpf_prog_type type;
enum bpf_attach_type expected_attach_type;
+
int prog_ifindex;
__u32 attach_btf_obj_fd;
__u32 attach_btf_id;
__u32 attach_prog_fd;
+
void *func_info;
__u32 func_info_rec_size;
__u32 func_info_cnt;
@@ -356,6 +467,14 @@ enum libbpf_map_type {
LIBBPF_MAP_KCONFIG,
};
+struct bpf_map_def {
+ unsigned int type;
+ unsigned int key_size;
+ unsigned int value_size;
+ unsigned int max_entries;
+ unsigned int map_flags;
+};
+
struct bpf_map {
struct bpf_object *obj;
char *name;
@@ -376,8 +495,6 @@ struct bpf_map {
__u32 btf_key_type_id;
__u32 btf_value_type_id;
__u32 btf_vmlinux_value_type_id;
- void *priv;
- bpf_map_clear_priv_t clear_priv;
enum libbpf_map_type libbpf_type;
void *mmaped;
struct bpf_struct_ops *st_ops;
@@ -440,8 +557,6 @@ struct extern_desc {
};
};
-static LIST_HEAD(bpf_objects_list);
-
struct module_btf {
struct btf *btf;
char *name;
@@ -510,12 +625,6 @@ struct bpf_object {
/* Information when doing ELF related work. Only valid if efile.elf is not NULL */
struct elf_state efile;
- /*
- * All loaded bpf_object are linked in a list, which is
- * hidden to caller. bpf_objects__<func> handlers deal with
- * all objects.
- */
- struct list_head list;
struct btf *btf;
struct btf_ext *btf_ext;
@@ -541,9 +650,6 @@ struct bpf_object {
size_t log_size;
__u32 log_level;
- void *priv;
- bpf_object_clear_priv_t clear_priv;
-
int *fd_array;
size_t fd_array_cap;
size_t fd_array_cnt;
@@ -565,25 +671,10 @@ static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
void bpf_program__unload(struct bpf_program *prog)
{
- int i;
-
if (!prog)
return;
- /*
- * If the object is opened but the program was never loaded,
- * it is possible that prog->instances.nr == -1.
- */
- if (prog->instances.nr > 0) {
- for (i = 0; i < prog->instances.nr; i++)
- zclose(prog->instances.fds[i]);
- } else if (prog->instances.nr != -1) {
- pr_warn("Internal error: instances.nr is %d\n",
- prog->instances.nr);
- }
-
- prog->instances.nr = -1;
- zfree(&prog->instances.fds);
+ zclose(prog->fd);
zfree(&prog->func_info);
zfree(&prog->line_info);
@@ -594,16 +685,9 @@ static void bpf_program__exit(struct bpf_program *prog)
if (!prog)
return;
- if (prog->clear_priv)
- prog->clear_priv(prog, prog->priv);
-
- prog->priv = NULL;
- prog->clear_priv = NULL;
-
bpf_program__unload(prog);
zfree(&prog->name);
zfree(&prog->sec_name);
- zfree(&prog->pin_name);
zfree(&prog->insns);
zfree(&prog->reloc_desc);
@@ -612,26 +696,6 @@ static void bpf_program__exit(struct bpf_program *prog)
prog->sec_idx = -1;
}
-static char *__bpf_program__pin_name(struct bpf_program *prog)
-{
- char *name, *p;
-
- if (libbpf_mode & LIBBPF_STRICT_SEC_NAME)
- name = strdup(prog->name);
- else
- name = strdup(prog->sec_name);
-
- if (!name)
- return NULL;
-
- p = name;
-
- while ((p = strchr(p, '/')))
- *p = '_';
-
- return name;
-}
-
static bool insn_is_subprog_call(const struct bpf_insn *insn)
{
return BPF_CLASS(insn->code) == BPF_JMP &&
@@ -673,6 +737,7 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
prog->insns_cnt = prog->sec_insn_cnt;
prog->type = BPF_PROG_TYPE_UNSPEC;
+ prog->fd = -1;
/* libbpf's convention for SEC("?abc...") is that it's just like
* SEC("abc...") but the corresponding bpf_program starts out with
@@ -686,9 +751,6 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
prog->autoload = true;
}
- prog->instances.fds = NULL;
- prog->instances.nr = -1;
-
/* inherit object's log_level */
prog->log_level = obj->log_level;
@@ -700,10 +762,6 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
if (!prog->name)
goto errout;
- prog->pin_name = __bpf_program__pin_name(prog);
- if (!prog->pin_name)
- goto errout;
-
prog->insns = malloc(insn_data_sz);
if (!prog->insns)
goto errout;
@@ -1185,7 +1243,6 @@ static struct bpf_object *bpf_object__new(const char *path,
size_t obj_buf_sz,
const char *obj_name)
{
- bool strict = (libbpf_mode & LIBBPF_STRICT_NO_OBJECT_LIST);
struct bpf_object *obj;
char *end;
@@ -1223,9 +1280,6 @@ static struct bpf_object *bpf_object__new(const char *path,
obj->kern_version = get_kernel_version();
obj->loaded = false;
- INIT_LIST_HEAD(&obj->list);
- if (!strict)
- list_add(&obj->list, &bpf_objects_list);
return obj;
}
@@ -1258,10 +1312,7 @@ static int bpf_object__elf_init(struct bpf_object *obj)
}
if (obj->efile.obj_buf_sz > 0) {
- /*
- * obj_buf should have been validated by
- * bpf_object__open_buffer().
- */
+ /* obj_buf should have been validated by bpf_object__open_mem(). */
elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
} else {
obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
@@ -1643,7 +1694,7 @@ static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
switch (ext->kcfg.type) {
case KCFG_BOOL:
if (value == 'm') {
- pr_warn("extern (kcfg) %s=%c should be tristate or char\n",
+ pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
ext->name, value);
return -EINVAL;
}
@@ -1664,7 +1715,7 @@ static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
case KCFG_INT:
case KCFG_CHAR_ARR:
default:
- pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n",
+ pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
ext->name, value);
return -EINVAL;
}
@@ -1678,7 +1729,8 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
size_t len;
if (ext->kcfg.type != KCFG_CHAR_ARR) {
- pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value);
+ pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
+ ext->name, value);
return -EINVAL;
}
@@ -1692,7 +1744,7 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
/* strip quotes */
len -= 2;
if (len >= ext->kcfg.sz) {
- pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
+ pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
ext->name, value, len, ext->kcfg.sz - 1);
len = ext->kcfg.sz - 1;
}
@@ -1749,13 +1801,20 @@ static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
__u64 value)
{
- if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
- pr_warn("extern (kcfg) %s=%llu should be integer\n",
+ if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
+ ext->kcfg.type != KCFG_BOOL) {
+ pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
ext->name, (unsigned long long)value);
return -EINVAL;
}
+ if (ext->kcfg.type == KCFG_BOOL && value > 1) {
+ pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
+ ext->name, (unsigned long long)value);
+ return -EINVAL;
+
+ }
if (!is_kcfg_value_in_range(ext, value)) {
- pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n",
+ pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
ext->name, (unsigned long long)value, ext->kcfg.sz);
return -ERANGE;
}
@@ -1819,16 +1878,19 @@ static int bpf_object__process_kconfig_line(struct bpf_object *obj,
/* assume integer */
err = parse_u64(value, &num);
if (err) {
- pr_warn("extern (kcfg) %s=%s should be integer\n",
- ext->name, value);
+ pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
return err;
}
+ if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
+ pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
+ return -EINVAL;
+ }
err = set_kcfg_value_num(ext, ext_val, num);
break;
}
if (err)
return err;
- pr_debug("extern (kcfg) %s=%s\n", ext->name, value);
+ pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
return 0;
}
@@ -1924,143 +1986,6 @@ static int bpf_object__init_kconfig_map(struct bpf_object *obj)
return 0;
}
-static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
-{
- Elf_Data *symbols = obj->efile.symbols;
- int i, map_def_sz = 0, nr_maps = 0, nr_syms;
- Elf_Data *data = NULL;
- Elf_Scn *scn;
-
- if (obj->efile.maps_shndx < 0)
- return 0;
-
- if (libbpf_mode & LIBBPF_STRICT_MAP_DEFINITIONS) {
- pr_warn("legacy map definitions in SEC(\"maps\") are not supported\n");
- return -EOPNOTSUPP;
- }
-
- if (!symbols)
- return -EINVAL;
-
- scn = elf_sec_by_idx(obj, obj->efile.maps_shndx);
- data = elf_sec_data(obj, scn);
- if (!scn || !data) {
- pr_warn("elf: failed to get legacy map definitions for %s\n",
- obj->path);
- return -EINVAL;
- }
-
- /*
- * Count number of maps. Each map has a name.
- * Array of maps is not supported: only the first element is
- * considered.
- *
- * TODO: Detect array of map and report error.
- */
- nr_syms = symbols->d_size / sizeof(Elf64_Sym);
- for (i = 0; i < nr_syms; i++) {
- Elf64_Sym *sym = elf_sym_by_idx(obj, i);
-
- if (sym->st_shndx != obj->efile.maps_shndx)
- continue;
- if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION)
- continue;
- nr_maps++;
- }
- /* Assume equally sized map definitions */
- pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n",
- nr_maps, data->d_size, obj->path);
-
- if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) {
- pr_warn("elf: unable to determine legacy map definition size in %s\n",
- obj->path);
- return -EINVAL;
- }
- map_def_sz = data->d_size / nr_maps;
-
- /* Fill obj->maps using data in "maps" section. */
- for (i = 0; i < nr_syms; i++) {
- Elf64_Sym *sym = elf_sym_by_idx(obj, i);
- const char *map_name;
- struct bpf_map_def *def;
- struct bpf_map *map;
-
- if (sym->st_shndx != obj->efile.maps_shndx)
- continue;
- if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION)
- continue;
-
- map = bpf_object__add_map(obj);
- if (IS_ERR(map))
- return PTR_ERR(map);
-
- map_name = elf_sym_str(obj, sym->st_name);
- if (!map_name) {
- pr_warn("failed to get map #%d name sym string for obj %s\n",
- i, obj->path);
- return -LIBBPF_ERRNO__FORMAT;
- }
-
- pr_warn("map '%s' (legacy): legacy map definitions are deprecated, use BTF-defined maps instead\n", map_name);
-
- if (ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
- pr_warn("map '%s' (legacy): static maps are not supported\n", map_name);
- return -ENOTSUP;
- }
-
- map->libbpf_type = LIBBPF_MAP_UNSPEC;
- map->sec_idx = sym->st_shndx;
- map->sec_offset = sym->st_value;
- pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
- map_name, map->sec_idx, map->sec_offset);
- if (sym->st_value + map_def_sz > data->d_size) {
- pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
- obj->path, map_name);
- return -EINVAL;
- }
-
- map->name = strdup(map_name);
- if (!map->name) {
- pr_warn("map '%s': failed to alloc map name\n", map_name);
- return -ENOMEM;
- }
- pr_debug("map %d is \"%s\"\n", i, map->name);
- def = (struct bpf_map_def *)(data->d_buf + sym->st_value);
- /*
- * If the definition of the map in the object file fits in
- * bpf_map_def, copy it. Any extra fields in our version
- * of bpf_map_def will default to zero as a result of the
- * calloc above.
- */
- if (map_def_sz <= sizeof(struct bpf_map_def)) {
- memcpy(&map->def, def, map_def_sz);
- } else {
- /*
- * Here the map structure being read is bigger than what
- * we expect, truncate if the excess bits are all zero.
- * If they are not zero, reject this map as
- * incompatible.
- */
- char *b;
-
- for (b = ((char *)def) + sizeof(struct bpf_map_def);
- b < ((char *)def) + map_def_sz; b++) {
- if (*b != 0) {
- pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n",
- obj->path, map_name);
- if (strict)
- return -EINVAL;
- }
- }
- memcpy(&map->def, def, sizeof(struct bpf_map_def));
- }
-
- /* btf info may not exist but fill it in if it does exist */
- (void) bpf_map_find_btf_info(obj, map);
- }
- return 0;
-}
-
const struct btf_type *
skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
{
@@ -2114,6 +2039,7 @@ static const char *__btf_kind_str(__u16 kind)
case BTF_KIND_FLOAT: return "float";
case BTF_KIND_DECL_TAG: return "decl_tag";
case BTF_KIND_TYPE_TAG: return "type_tag";
+ case BTF_KIND_ENUM64: return "enum64";
default: return "unknown";
}
}
@@ -2177,6 +2103,13 @@ static int build_map_pin_path(struct bpf_map *map, const char *path)
return bpf_map__set_pin_path(map, buf);
}
+/* should match definition in bpf_helpers.h */
+enum libbpf_pin_type {
+ LIBBPF_PIN_NONE,
+ /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
+ LIBBPF_PIN_BY_NAME,
+};
+
int parse_btf_map_def(const char *map_name, struct btf *btf,
const struct btf_type *def_t, bool strict,
struct btf_map_def *map_def, struct btf_map_def *inner_def)
@@ -2398,6 +2331,37 @@ int parse_btf_map_def(const char *map_name, struct btf *btf,
return 0;
}
+static size_t adjust_ringbuf_sz(size_t sz)
+{
+ __u32 page_sz = sysconf(_SC_PAGE_SIZE);
+ __u32 mul;
+
+ /* if user forgot to set any size, make sure they see error */
+ if (sz == 0)
+ return 0;
+ /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
+ * a power-of-2 multiple of kernel's page size. If user diligently
+ * satisified these conditions, pass the size through.
+ */
+ if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
+ return sz;
+
+ /* Otherwise find closest (page_sz * power_of_2) product bigger than
+ * user-set size to satisfy both user size request and kernel
+ * requirements and substitute correct max_entries for map creation.
+ */
+ for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
+ if (mul * page_sz > sz)
+ return mul * page_sz;
+ }
+
+ /* if it's impossible to satisfy the conditions (i.e., user size is
+ * very close to UINT_MAX but is not a power-of-2 multiple of
+ * page_size) then just return original size and let kernel reject it
+ */
+ return sz;
+}
+
static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
{
map->def.type = def->map_type;
@@ -2411,6 +2375,10 @@ static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def
map->btf_key_type_id = def->key_type_id;
map->btf_value_type_id = def->value_type_id;
+ /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
+ if (map->def.type == BPF_MAP_TYPE_RINGBUF)
+ map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
+
if (def->parts & MAP_DEF_MAP_TYPE)
pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
@@ -2609,12 +2577,11 @@ static int bpf_object__init_maps(struct bpf_object *obj,
{
const char *pin_root_path;
bool strict;
- int err;
+ int err = 0;
strict = !OPTS_GET(opts, relaxed_maps, false);
pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
- err = bpf_object__init_user_maps(obj, strict);
err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
err = err ?: bpf_object__init_global_data_maps(obj);
err = err ?: bpf_object__init_kconfig_map(obj);
@@ -2642,12 +2609,13 @@ static bool btf_needs_sanitization(struct bpf_object *obj)
bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
+ bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
return !has_func || !has_datasec || !has_func_global || !has_float ||
- !has_decl_tag || !has_type_tag;
+ !has_decl_tag || !has_type_tag || !has_enum64;
}
-static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
+static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
{
bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
@@ -2655,6 +2623,8 @@ static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
+ bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
+ int enum64_placeholder_id = 0;
struct btf_type *t;
int i, j, vlen;
@@ -2717,8 +2687,32 @@ static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
/* replace TYPE_TAG with a CONST */
t->name_off = 0;
t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
- }
+ } else if (!has_enum64 && btf_is_enum(t)) {
+ /* clear the kflag */
+ t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
+ } else if (!has_enum64 && btf_is_enum64(t)) {
+ /* replace ENUM64 with a union */
+ struct btf_member *m;
+
+ if (enum64_placeholder_id == 0) {
+ enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
+ if (enum64_placeholder_id < 0)
+ return enum64_placeholder_id;
+
+ t = (struct btf_type *)btf__type_by_id(btf, i);
+ }
+
+ m = btf_members(t);
+ vlen = btf_vlen(t);
+ t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
+ for (j = 0; j < vlen; j++, m++) {
+ m->type = enum64_placeholder_id;
+ m->offset = 0;
+ }
+ }
}
+
+ return 0;
}
static bool libbpf_needs_btf(const struct bpf_object *obj)
@@ -2905,11 +2899,6 @@ static int btf_finalize_data(struct bpf_object *obj, struct btf *btf)
return libbpf_err(err);
}
-int btf__finalize_data(struct bpf_object *obj, struct btf *btf)
-{
- return btf_finalize_data(obj, btf);
-}
-
static int bpf_object__finalize_btf(struct bpf_object *obj)
{
int err;
@@ -3056,7 +3045,9 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
/* enforce 8-byte pointers for BPF-targeted BTFs */
btf__set_pointer_size(obj->btf, 8);
- bpf_object__sanitize_btf(obj, kern_btf);
+ err = bpf_object__sanitize_btf(obj, kern_btf);
+ if (err)
+ return err;
}
if (obj->gen_loader) {
@@ -3563,6 +3554,10 @@ static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
if (strcmp(name, "libbpf_tristate"))
return KCFG_UNKNOWN;
return KCFG_TRISTATE;
+ case BTF_KIND_ENUM64:
+ if (strcmp(name, "libbpf_tristate"))
+ return KCFG_UNKNOWN;
+ return KCFG_TRISTATE;
case BTF_KIND_ARRAY:
if (btf_array(t)->nelems == 0)
return KCFG_UNKNOWN;
@@ -3738,7 +3733,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj)
ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
&ext->kcfg.is_signed);
if (ext->kcfg.type == KCFG_UNKNOWN) {
- pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name);
+ pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
return -ENOTSUP;
}
} else if (strcmp(sec_name, KSYMS_SEC) == 0) {
@@ -3860,41 +3855,8 @@ static int bpf_object__collect_externs(struct bpf_object *obj)
return 0;
}
-struct bpf_program *
-bpf_object__find_program_by_title(const struct bpf_object *obj,
- const char *title)
+static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
{
- struct bpf_program *pos;
-
- bpf_object__for_each_program(pos, obj) {
- if (pos->sec_name && !strcmp(pos->sec_name, title))
- return pos;
- }
- return errno = ENOENT, NULL;
-}
-
-static bool prog_is_subprog(const struct bpf_object *obj,
- const struct bpf_program *prog)
-{
- /* For legacy reasons, libbpf supports an entry-point BPF programs
- * without SEC() attribute, i.e., those in the .text section. But if
- * there are 2 or more such programs in the .text section, they all
- * must be subprograms called from entry-point BPF programs in
- * designated SEC()'tions, otherwise there is no way to distinguish
- * which of those programs should be loaded vs which are a subprogram.
- * Similarly, if there is a function/program in .text and at least one
- * other BPF program with custom SEC() attribute, then we just assume
- * .text programs are subprograms (even if they are not called from
- * other programs), because libbpf never explicitly supported mixing
- * SEC()-designated BPF programs and .text entry-point BPF programs.
- *
- * In libbpf 1.0 strict mode, we always consider .text
- * programs to be subprograms.
- */
-
- if (libbpf_mode & LIBBPF_STRICT_SEC_NAME)
- return prog->sec_idx == obj->efile.text_shndx;
-
return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
}
@@ -4235,9 +4197,7 @@ bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Dat
static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
{
- struct bpf_map_def *def = &map->def;
- __u32 key_type_id = 0, value_type_id = 0;
- int ret;
+ int id;
if (!obj->btf)
return -ENOENT;
@@ -4246,31 +4206,22 @@ static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map)
* For struct_ops map, it does not need btf_key_type_id and
* btf_value_type_id.
*/
- if (map->sec_idx == obj->efile.btf_maps_shndx ||
- bpf_map__is_struct_ops(map))
+ if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
return 0;
- if (!bpf_map__is_internal(map)) {
- pr_warn("Use of BPF_ANNOTATE_KV_PAIR is deprecated, use BTF-defined maps in .maps section instead\n");
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
- ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
- def->value_size, &key_type_id,
- &value_type_id);
-#pragma GCC diagnostic pop
- } else {
- /*
- * LLVM annotates global data differently in BTF, that is,
- * only as '.data', '.bss' or '.rodata'.
- */
- ret = btf__find_by_name(obj->btf, map->real_name);
- }
- if (ret < 0)
- return ret;
+ /*
+ * LLVM annotates global data differently in BTF, that is,
+ * only as '.data', '.bss' or '.rodata'.
+ */
+ if (!bpf_map__is_internal(map))
+ return -ENOENT;
- map->btf_key_type_id = key_type_id;
- map->btf_value_type_id = bpf_map__is_internal(map) ?
- ret : value_type_id;
+ id = btf__find_by_name(obj->btf, map->real_name);
+ if (id < 0)
+ return id;
+
+ map->btf_key_type_id = 0;
+ map->btf_value_type_id = id;
return 0;
}
@@ -4327,7 +4278,7 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
int bpf_map__reuse_fd(struct bpf_map *map, int fd)
{
struct bpf_map_info info = {};
- __u32 len = sizeof(info);
+ __u32 len = sizeof(info), name_len;
int new_fd, err;
char *new_name;
@@ -4337,7 +4288,12 @@ int bpf_map__reuse_fd(struct bpf_map *map, int fd)
if (err)
return libbpf_err(err);
- new_name = strdup(info.name);
+ name_len = strlen(info.name);
+ if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
+ new_name = strdup(map->name);
+ else
+ new_name = strdup(info.name);
+
if (!new_name)
return libbpf_err(-errno);
@@ -4396,18 +4352,16 @@ struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
{
- if (map->fd >= 0)
+ if (map->obj->loaded)
return libbpf_err(-EBUSY);
+
map->def.max_entries = max_entries;
- return 0;
-}
-int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
-{
- if (!map || !max_entries)
- return libbpf_err(-EINVAL);
+ /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
+ if (map->def.type == BPF_MAP_TYPE_RINGBUF)
+ map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
- return bpf_map__set_max_entries(map, max_entries);
+ return 0;
}
static int
@@ -4746,6 +4700,19 @@ static int probe_kern_bpf_cookie(void)
return probe_fd(ret);
}
+static int probe_kern_btf_enum64(void)
+{
+ static const char strs[] = "\0enum64";
+ __u32 types[] = {
+ BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
+ };
+
+ return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
+ strs, sizeof(strs)));
+}
+
+static int probe_kern_syscall_wrapper(void);
+
enum kern_feature_result {
FEAT_UNKNOWN = 0,
FEAT_SUPPORTED = 1,
@@ -4811,6 +4778,12 @@ static struct kern_feature_desc {
[FEAT_BPF_COOKIE] = {
"BPF cookie support", probe_kern_bpf_cookie,
},
+ [FEAT_BTF_ENUM64] = {
+ "BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
+ },
+ [FEAT_SYSCALL_WRAPPER] = {
+ "Kernel using syscall wrapper", probe_kern_syscall_wrapper,
+ },
};
bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
@@ -4943,42 +4916,6 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
static void bpf_map__destroy(struct bpf_map *map);
-static bool is_pow_of_2(size_t x)
-{
- return x && (x & (x - 1));
-}
-
-static size_t adjust_ringbuf_sz(size_t sz)
-{
- __u32 page_sz = sysconf(_SC_PAGE_SIZE);
- __u32 mul;
-
- /* if user forgot to set any size, make sure they see error */
- if (sz == 0)
- return 0;
- /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
- * a power-of-2 multiple of kernel's page size. If user diligently
- * satisified these conditions, pass the size through.
- */
- if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
- return sz;
-
- /* Otherwise find closest (page_sz * power_of_2) product bigger than
- * user-set size to satisfy both user size request and kernel
- * requirements and substitute correct max_entries for map creation.
- */
- for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
- if (mul * page_sz > sz)
- return mul * page_sz;
- }
-
- /* if it's impossible to satisfy the conditions (i.e., user size is
- * very close to UINT_MAX but is not a power-of-2 multiple of
- * page_size) then just return original size and let kernel reject it
- */
- return sz;
-}
-
static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
{
LIBBPF_OPTS(bpf_map_create_opts, create_attr);
@@ -5017,9 +4954,6 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b
}
switch (def->type) {
- case BPF_MAP_TYPE_RINGBUF:
- map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
- /* fallthrough */
case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
case BPF_MAP_TYPE_CGROUP_ARRAY:
case BPF_MAP_TYPE_STACK_TRACE:
@@ -5353,7 +5287,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand,
n = btf__type_cnt(targ_btf);
for (i = targ_start_id; i < n; i++) {
t = btf__type_by_id(targ_btf, i);
- if (btf_kind(t) != btf_kind(local_t))
+ if (!btf_kind_core_compat(t, local_t))
continue;
targ_name = btf__name_by_offset(targ_btf, t->name_off);
@@ -5561,76 +5495,13 @@ err_out:
int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
const struct btf *targ_btf, __u32 targ_id)
{
- const struct btf_type *local_type, *targ_type;
- int depth = 32; /* max recursion depth */
-
- /* caller made sure that names match (ignoring flavor suffix) */
- local_type = btf__type_by_id(local_btf, local_id);
- targ_type = btf__type_by_id(targ_btf, targ_id);
- if (btf_kind(local_type) != btf_kind(targ_type))
- return 0;
-
-recur:
- depth--;
- if (depth < 0)
- return -EINVAL;
-
- local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
- targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
- if (!local_type || !targ_type)
- return -EINVAL;
-
- if (btf_kind(local_type) != btf_kind(targ_type))
- return 0;
-
- switch (btf_kind(local_type)) {
- case BTF_KIND_UNKN:
- case BTF_KIND_STRUCT:
- case BTF_KIND_UNION:
- case BTF_KIND_ENUM:
- case BTF_KIND_FWD:
- return 1;
- case BTF_KIND_INT:
- /* just reject deprecated bitfield-like integers; all other
- * integers are by default compatible between each other
- */
- return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
- case BTF_KIND_PTR:
- local_id = local_type->type;
- targ_id = targ_type->type;
- goto recur;
- case BTF_KIND_ARRAY:
- local_id = btf_array(local_type)->type;
- targ_id = btf_array(targ_type)->type;
- goto recur;
- case BTF_KIND_FUNC_PROTO: {
- struct btf_param *local_p = btf_params(local_type);
- struct btf_param *targ_p = btf_params(targ_type);
- __u16 local_vlen = btf_vlen(local_type);
- __u16 targ_vlen = btf_vlen(targ_type);
- int i, err;
-
- if (local_vlen != targ_vlen)
- return 0;
-
- for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
- skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
- skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
- err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id);
- if (err <= 0)
- return err;
- }
+ return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32);
+}
- /* tail recurse for return type check */
- skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
- skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
- goto recur;
- }
- default:
- pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
- btf_kind_str(local_type), local_id, targ_id);
- return 0;
- }
+int bpf_core_types_match(const struct btf *local_btf, __u32 local_id,
+ const struct btf *targ_btf, __u32 targ_id)
+{
+ return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32);
}
static size_t bpf_core_hash_fn(const void *key, void *ctx)
@@ -6754,11 +6625,6 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog,
if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
- if (def & SEC_DEPRECATED) {
- pr_warn("SEC(\"%s\") is deprecated, please see https://github.com/libbpf/libbpf/wiki/Libbpf-1.0-migration-guide#bpf-program-sec-annotation-deprecations for details\n",
- prog->sec_name);
- }
-
if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
int btf_obj_fd = 0, btf_type_id = 0, err;
const char *attach_name;
@@ -6801,10 +6667,9 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog,
static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
-static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_program *prog,
- struct bpf_insn *insns, int insns_cnt,
- const char *license, __u32 kern_version,
- int *prog_fd)
+static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
+ struct bpf_insn *insns, int insns_cnt,
+ const char *license, __u32 kern_version, int *prog_fd)
{
LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
const char *prog_name = NULL;
@@ -7171,93 +7036,6 @@ static int bpf_program_record_relos(struct bpf_program *prog)
return 0;
}
-static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
- const char *license, __u32 kern_ver)
-{
- int err = 0, fd, i;
-
- if (obj->loaded) {
- pr_warn("prog '%s': can't load after object was loaded\n", prog->name);
- return libbpf_err(-EINVAL);
- }
-
- if (prog->instances.nr < 0 || !prog->instances.fds) {
- if (prog->preprocessor) {
- pr_warn("Internal error: can't load program '%s'\n",
- prog->name);
- return libbpf_err(-LIBBPF_ERRNO__INTERNAL);
- }
-
- prog->instances.fds = malloc(sizeof(int));
- if (!prog->instances.fds) {
- pr_warn("Not enough memory for BPF fds\n");
- return libbpf_err(-ENOMEM);
- }
- prog->instances.nr = 1;
- prog->instances.fds[0] = -1;
- }
-
- if (!prog->preprocessor) {
- if (prog->instances.nr != 1) {
- pr_warn("prog '%s': inconsistent nr(%d) != 1\n",
- prog->name, prog->instances.nr);
- }
- if (obj->gen_loader)
- bpf_program_record_relos(prog);
- err = bpf_object_load_prog_instance(obj, prog,
- prog->insns, prog->insns_cnt,
- license, kern_ver, &fd);
- if (!err)
- prog->instances.fds[0] = fd;
- goto out;
- }
-
- for (i = 0; i < prog->instances.nr; i++) {
- struct bpf_prog_prep_result result;
- bpf_program_prep_t preprocessor = prog->preprocessor;
-
- memset(&result, 0, sizeof(result));
- err = preprocessor(prog, i, prog->insns,
- prog->insns_cnt, &result);
- if (err) {
- pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
- i, prog->name);
- goto out;
- }
-
- if (!result.new_insn_ptr || !result.new_insn_cnt) {
- pr_debug("Skip loading the %dth instance of program '%s'\n",
- i, prog->name);
- prog->instances.fds[i] = -1;
- if (result.pfd)
- *result.pfd = -1;
- continue;
- }
-
- err = bpf_object_load_prog_instance(obj, prog,
- result.new_insn_ptr, result.new_insn_cnt,
- license, kern_ver, &fd);
- if (err) {
- pr_warn("Loading the %dth instance of program '%s' failed\n",
- i, prog->name);
- goto out;
- }
-
- if (result.pfd)
- *result.pfd = fd;
- prog->instances.fds[i] = fd;
- }
-out:
- if (err)
- pr_warn("failed to load program '%s'\n", prog->name);
- return libbpf_err(err);
-}
-
-int bpf_program__load(struct bpf_program *prog, const char *license, __u32 kern_ver)
-{
- return bpf_object_load_prog(prog->obj, prog, license, kern_ver);
-}
-
static int
bpf_object__load_progs(struct bpf_object *obj, int log_level)
{
@@ -7281,9 +7059,16 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level)
continue;
}
prog->log_level |= log_level;
- err = bpf_object_load_prog(obj, prog, obj->license, obj->kern_version);
- if (err)
+
+ if (obj->gen_loader)
+ bpf_program_record_relos(prog);
+
+ err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
+ obj->license, obj->kern_version, &prog->fd);
+ if (err) {
+ pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
return err;
+ }
}
bpf_object__free_relocs(obj);
@@ -7309,13 +7094,6 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object
prog->type = prog->sec_def->prog_type;
prog->expected_attach_type = prog->sec_def->expected_attach_type;
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
- if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
- prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
- prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
-#pragma GCC diagnostic pop
-
/* sec_def can have custom callback which should be called
* after bpf_program is initialized to adjust its properties
*/
@@ -7421,36 +7199,6 @@ out:
return ERR_PTR(err);
}
-static struct bpf_object *
-__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
-{
- DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
- .relaxed_maps = flags & MAPS_RELAX_COMPAT,
- );
-
- /* param validation */
- if (!attr->file)
- return NULL;
-
- pr_debug("loading %s\n", attr->file);
- return bpf_object_open(attr->file, NULL, 0, &opts);
-}
-
-struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
-{
- return libbpf_ptr(__bpf_object__open_xattr(attr, 0));
-}
-
-struct bpf_object *bpf_object__open(const char *path)
-{
- struct bpf_object_open_attr attr = {
- .file = path,
- .prog_type = BPF_PROG_TYPE_UNSPEC,
- };
-
- return libbpf_ptr(__bpf_object__open_xattr(&attr, 0));
-}
-
struct bpf_object *
bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
{
@@ -7462,6 +7210,11 @@ bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts)
return libbpf_ptr(bpf_object_open(path, NULL, 0, opts));
}
+struct bpf_object *bpf_object__open(const char *path)
+{
+ return bpf_object__open_file(path, NULL);
+}
+
struct bpf_object *
bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
const struct bpf_object_open_opts *opts)
@@ -7472,23 +7225,6 @@ bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts));
}
-struct bpf_object *
-bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
- const char *name)
-{
- DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
- .object_name = name,
- /* wrong default, but backwards-compatible */
- .relaxed_maps = true,
- );
-
- /* returning NULL is wrong, but backwards-compatible */
- if (!obj_buf || obj_buf_sz == 0)
- return errno = EINVAL, NULL;
-
- return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, &opts));
-}
-
static int bpf_object_unload(struct bpf_object *obj)
{
size_t i;
@@ -7574,14 +7310,14 @@ static int kallsyms_cb(unsigned long long sym_addr, char sym_type,
return 0;
if (ext->is_set && ext->ksym.addr != sym_addr) {
- pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n",
+ pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n",
sym_name, ext->ksym.addr, sym_addr);
return -EINVAL;
}
if (!ext->is_set) {
ext->is_set = true;
ext->ksym.addr = sym_addr;
- pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr);
+ pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr);
}
return 0;
}
@@ -7785,28 +7521,52 @@ static int bpf_object__resolve_externs(struct bpf_object *obj,
for (i = 0; i < obj->nr_extern; i++) {
ext = &obj->externs[i];
- if (ext->type == EXT_KCFG &&
- strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
- void *ext_val = kcfg_data + ext->kcfg.data_off;
- __u32 kver = get_kernel_version();
+ if (ext->type == EXT_KSYM) {
+ if (ext->ksym.type_id)
+ need_vmlinux_btf = true;
+ else
+ need_kallsyms = true;
+ continue;
+ } else if (ext->type == EXT_KCFG) {
+ void *ext_ptr = kcfg_data + ext->kcfg.data_off;
+ __u64 value = 0;
+
+ /* Kconfig externs need actual /proc/config.gz */
+ if (str_has_pfx(ext->name, "CONFIG_")) {
+ need_config = true;
+ continue;
+ }
- if (!kver) {
- pr_warn("failed to get kernel version\n");
+ /* Virtual kcfg externs are customly handled by libbpf */
+ if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {
+ value = get_kernel_version();
+ if (!value) {
+ pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);
+ return -EINVAL;
+ }
+ } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {
+ value = kernel_supports(obj, FEAT_BPF_COOKIE);
+ } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {
+ value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
+ } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {
+ /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed
+ * __kconfig externs, where LINUX_ ones are virtual and filled out
+ * customly by libbpf (their values don't come from Kconfig).
+ * If LINUX_xxx variable is not recognized by libbpf, but is marked
+ * __weak, it defaults to zero value, just like for CONFIG_xxx
+ * externs.
+ */
+ pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);
return -EINVAL;
}
- err = set_kcfg_value_num(ext, ext_val, kver);
+
+ err = set_kcfg_value_num(ext, ext_ptr, value);
if (err)
return err;
- pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver);
- } else if (ext->type == EXT_KCFG && str_has_pfx(ext->name, "CONFIG_")) {
- need_config = true;
- } else if (ext->type == EXT_KSYM) {
- if (ext->ksym.type_id)
- need_vmlinux_btf = true;
- else
- need_kallsyms = true;
+ pr_debug("extern (kcfg) '%s': set to 0x%llx\n",
+ ext->name, (long long)value);
} else {
- pr_warn("unrecognized extern '%s'\n", ext->name);
+ pr_warn("extern '%s': unrecognized extern kind\n", ext->name);
return -EINVAL;
}
}
@@ -7842,10 +7602,10 @@ static int bpf_object__resolve_externs(struct bpf_object *obj,
ext = &obj->externs[i];
if (!ext->is_set && !ext->is_weak) {
- pr_warn("extern %s (strong) not resolved\n", ext->name);
+ pr_warn("extern '%s' (strong): not resolved\n", ext->name);
return -ESRCH;
} else if (!ext->is_set) {
- pr_debug("extern %s (weak) not resolved, defaulting to zero\n",
+ pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n",
ext->name);
}
}
@@ -7921,11 +7681,6 @@ out:
return libbpf_err(err);
}
-int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
-{
- return bpf_object_load(attr->obj, attr->log_level, attr->target_btf_path);
-}
-
int bpf_object__load(struct bpf_object *obj)
{
return bpf_object_load(obj, 0, NULL);
@@ -7983,11 +7738,16 @@ static int check_path(const char *path)
return err;
}
-static int bpf_program_pin_instance(struct bpf_program *prog, const char *path, int instance)
+int bpf_program__pin(struct bpf_program *prog, const char *path)
{
char *cp, errmsg[STRERR_BUFSIZE];
int err;
+ if (prog->fd < 0) {
+ pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
+ return libbpf_err(-EINVAL);
+ }
+
err = make_parent_dir(path);
if (err)
return libbpf_err(err);
@@ -7996,170 +7756,35 @@ static int bpf_program_pin_instance(struct bpf_program *prog, const char *path,
if (err)
return libbpf_err(err);
- if (prog == NULL) {
- pr_warn("invalid program pointer\n");
- return libbpf_err(-EINVAL);
- }
-
- if (instance < 0 || instance >= prog->instances.nr) {
- pr_warn("invalid prog instance %d of prog %s (max %d)\n",
- instance, prog->name, prog->instances.nr);
- return libbpf_err(-EINVAL);
- }
-
- if (bpf_obj_pin(prog->instances.fds[instance], path)) {
+ if (bpf_obj_pin(prog->fd, path)) {
err = -errno;
cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
- pr_warn("failed to pin program: %s\n", cp);
+ pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
return libbpf_err(err);
}
- pr_debug("pinned program '%s'\n", path);
+ pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
return 0;
}
-static int bpf_program_unpin_instance(struct bpf_program *prog, const char *path, int instance)
+int bpf_program__unpin(struct bpf_program *prog, const char *path)
{
int err;
- err = check_path(path);
- if (err)
- return libbpf_err(err);
-
- if (prog == NULL) {
- pr_warn("invalid program pointer\n");
- return libbpf_err(-EINVAL);
- }
-
- if (instance < 0 || instance >= prog->instances.nr) {
- pr_warn("invalid prog instance %d of prog %s (max %d)\n",
- instance, prog->name, prog->instances.nr);
+ if (prog->fd < 0) {
+ pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
return libbpf_err(-EINVAL);
}
- err = unlink(path);
- if (err != 0)
- return libbpf_err(-errno);
-
- pr_debug("unpinned program '%s'\n", path);
-
- return 0;
-}
-
-__attribute__((alias("bpf_program_pin_instance")))
-int bpf_object__pin_instance(struct bpf_program *prog, const char *path, int instance);
-
-__attribute__((alias("bpf_program_unpin_instance")))
-int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, int instance);
-
-int bpf_program__pin(struct bpf_program *prog, const char *path)
-{
- int i, err;
-
- err = make_parent_dir(path);
- if (err)
- return libbpf_err(err);
-
- err = check_path(path);
- if (err)
- return libbpf_err(err);
-
- if (prog == NULL) {
- pr_warn("invalid program pointer\n");
- return libbpf_err(-EINVAL);
- }
-
- if (prog->instances.nr <= 0) {
- pr_warn("no instances of prog %s to pin\n", prog->name);
- return libbpf_err(-EINVAL);
- }
-
- if (prog->instances.nr == 1) {
- /* don't create subdirs when pinning single instance */
- return bpf_program_pin_instance(prog, path, 0);
- }
-
- for (i = 0; i < prog->instances.nr; i++) {
- char buf[PATH_MAX];
- int len;
-
- len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
- if (len < 0) {
- err = -EINVAL;
- goto err_unpin;
- } else if (len >= PATH_MAX) {
- err = -ENAMETOOLONG;
- goto err_unpin;
- }
-
- err = bpf_program_pin_instance(prog, buf, i);
- if (err)
- goto err_unpin;
- }
-
- return 0;
-
-err_unpin:
- for (i = i - 1; i >= 0; i--) {
- char buf[PATH_MAX];
- int len;
-
- len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
- if (len < 0)
- continue;
- else if (len >= PATH_MAX)
- continue;
-
- bpf_program_unpin_instance(prog, buf, i);
- }
-
- rmdir(path);
-
- return libbpf_err(err);
-}
-
-int bpf_program__unpin(struct bpf_program *prog, const char *path)
-{
- int i, err;
-
err = check_path(path);
if (err)
return libbpf_err(err);
- if (prog == NULL) {
- pr_warn("invalid program pointer\n");
- return libbpf_err(-EINVAL);
- }
-
- if (prog->instances.nr <= 0) {
- pr_warn("no instances of prog %s to pin\n", prog->name);
- return libbpf_err(-EINVAL);
- }
-
- if (prog->instances.nr == 1) {
- /* don't create subdirs when pinning single instance */
- return bpf_program_unpin_instance(prog, path, 0);
- }
-
- for (i = 0; i < prog->instances.nr; i++) {
- char buf[PATH_MAX];
- int len;
-
- len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
- if (len < 0)
- return libbpf_err(-EINVAL);
- else if (len >= PATH_MAX)
- return libbpf_err(-ENAMETOOLONG);
-
- err = bpf_program_unpin_instance(prog, buf, i);
- if (err)
- return err;
- }
-
- err = rmdir(path);
+ err = unlink(path);
if (err)
return libbpf_err(-errno);
+ pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
return 0;
}
@@ -8406,8 +8031,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
char buf[PATH_MAX];
int len;
- len = snprintf(buf, PATH_MAX, "%s/%s", path,
- prog->pin_name);
+ len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
if (len < 0) {
err = -EINVAL;
goto err_unpin_programs;
@@ -8428,8 +8052,7 @@ err_unpin_programs:
char buf[PATH_MAX];
int len;
- len = snprintf(buf, PATH_MAX, "%s/%s", path,
- prog->pin_name);
+ len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
if (len < 0)
continue;
else if (len >= PATH_MAX)
@@ -8453,8 +8076,7 @@ int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
char buf[PATH_MAX];
int len;
- len = snprintf(buf, PATH_MAX, "%s/%s", path,
- prog->pin_name);
+ len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
if (len < 0)
return libbpf_err(-EINVAL);
else if (len >= PATH_MAX)
@@ -8487,11 +8109,6 @@ int bpf_object__pin(struct bpf_object *obj, const char *path)
static void bpf_map__destroy(struct bpf_map *map)
{
- if (map->clear_priv)
- map->clear_priv(map, map->priv);
- map->priv = NULL;
- map->clear_priv = NULL;
-
if (map->inner_map) {
bpf_map__destroy(map->inner_map);
zfree(&map->inner_map);
@@ -8527,9 +8144,6 @@ void bpf_object__close(struct bpf_object *obj)
if (IS_ERR_OR_NULL(obj))
return;
- if (obj->clear_priv)
- obj->clear_priv(obj, obj->priv);
-
usdt_manager_free(obj->usdt_man);
obj->usdt_man = NULL;
@@ -8556,33 +8170,9 @@ void bpf_object__close(struct bpf_object *obj)
}
zfree(&obj->programs);
- list_del(&obj->list);
free(obj);
}
-struct bpf_object *
-bpf_object__next(struct bpf_object *prev)
-{
- struct bpf_object *next;
- bool strict = (libbpf_mode & LIBBPF_STRICT_NO_OBJECT_LIST);
-
- if (strict)
- return NULL;
-
- if (!prev)
- next = list_first_entry(&bpf_objects_list,
- struct bpf_object,
- list);
- else
- next = list_next_entry(prev, list);
-
- /* Empty list is noticed here so don't need checking on entry. */
- if (&next->list == &bpf_objects_list)
- return NULL;
-
- return next;
-}
-
const char *bpf_object__name(const struct bpf_object *obj)
{
return obj ? obj->name : libbpf_err_ptr(-EINVAL);
@@ -8613,22 +8203,6 @@ int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
return 0;
}
-int bpf_object__set_priv(struct bpf_object *obj, void *priv,
- bpf_object_clear_priv_t clear_priv)
-{
- if (obj->priv && obj->clear_priv)
- obj->clear_priv(obj, obj->priv);
-
- obj->priv = priv;
- obj->clear_priv = clear_priv;
- return 0;
-}
-
-void *bpf_object__priv(const struct bpf_object *obj)
-{
- return obj ? obj->priv : libbpf_err_ptr(-EINVAL);
-}
-
int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
{
struct bpf_gen *gen;
@@ -8672,12 +8246,6 @@ __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
}
struct bpf_program *
-bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj)
-{
- return bpf_object__next_program(obj, prev);
-}
-
-struct bpf_program *
bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
{
struct bpf_program *prog = prev;
@@ -8690,12 +8258,6 @@ bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
}
struct bpf_program *
-bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj)
-{
- return bpf_object__prev_program(obj, next);
-}
-
-struct bpf_program *
bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
{
struct bpf_program *prog = next;
@@ -8707,22 +8269,6 @@ bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
return prog;
}
-int bpf_program__set_priv(struct bpf_program *prog, void *priv,
- bpf_program_clear_priv_t clear_priv)
-{
- if (prog->priv && prog->clear_priv)
- prog->clear_priv(prog, prog->priv);
-
- prog->priv = priv;
- prog->clear_priv = clear_priv;
- return 0;
-}
-
-void *bpf_program__priv(const struct bpf_program *prog)
-{
- return prog ? prog->priv : libbpf_err_ptr(-EINVAL);
-}
-
void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
{
prog->prog_ifindex = ifindex;
@@ -8738,22 +8284,6 @@ const char *bpf_program__section_name(const struct bpf_program *prog)
return prog->sec_name;
}
-const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy)
-{
- const char *title;
-
- title = prog->sec_name;
- if (needs_copy) {
- title = strdup(title);
- if (!title) {
- pr_warn("failed to strdup program title\n");
- return libbpf_err_ptr(-ENOMEM);
- }
- }
-
- return title;
-}
-
bool bpf_program__autoload(const struct bpf_program *prog)
{
return prog->autoload;
@@ -8768,18 +8298,6 @@ int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
return 0;
}
-static int bpf_program_nth_fd(const struct bpf_program *prog, int n);
-
-int bpf_program__fd(const struct bpf_program *prog)
-{
- return bpf_program_nth_fd(prog, 0);
-}
-
-size_t bpf_program__size(const struct bpf_program *prog)
-{
- return prog->insns_cnt * BPF_INSN_SZ;
-}
-
const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
{
return prog->insns;
@@ -8810,58 +8328,15 @@ int bpf_program__set_insns(struct bpf_program *prog,
return 0;
}
-int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
- bpf_program_prep_t prep)
-{
- int *instances_fds;
-
- if (nr_instances <= 0 || !prep)
- return libbpf_err(-EINVAL);
-
- if (prog->instances.nr > 0 || prog->instances.fds) {
- pr_warn("Can't set pre-processor after loading\n");
- return libbpf_err(-EINVAL);
- }
-
- instances_fds = malloc(sizeof(int) * nr_instances);
- if (!instances_fds) {
- pr_warn("alloc memory failed for fds\n");
- return libbpf_err(-ENOMEM);
- }
-
- /* fill all fd with -1 */
- memset(instances_fds, -1, sizeof(int) * nr_instances);
-
- prog->instances.nr = nr_instances;
- prog->instances.fds = instances_fds;
- prog->preprocessor = prep;
- return 0;
-}
-
-__attribute__((alias("bpf_program_nth_fd")))
-int bpf_program__nth_fd(const struct bpf_program *prog, int n);
-
-static int bpf_program_nth_fd(const struct bpf_program *prog, int n)
+int bpf_program__fd(const struct bpf_program *prog)
{
- int fd;
-
if (!prog)
return libbpf_err(-EINVAL);
- if (n >= prog->instances.nr || n < 0) {
- pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
- n, prog->name, prog->instances.nr);
- return libbpf_err(-EINVAL);
- }
-
- fd = prog->instances.fds[n];
- if (fd < 0) {
- pr_warn("%dth instance of program '%s' is invalid\n",
- n, prog->name);
+ if (prog->fd < 0)
return libbpf_err(-ENOENT);
- }
- return fd;
+ return prog->fd;
}
__alias(bpf_program__type)
@@ -8881,39 +8356,6 @@ int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
return 0;
}
-static bool bpf_program__is_type(const struct bpf_program *prog,
- enum bpf_prog_type type)
-{
- return prog ? (prog->type == type) : false;
-}
-
-#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
-int bpf_program__set_##NAME(struct bpf_program *prog) \
-{ \
- if (!prog) \
- return libbpf_err(-EINVAL); \
- return bpf_program__set_type(prog, TYPE); \
-} \
- \
-bool bpf_program__is_##NAME(const struct bpf_program *prog) \
-{ \
- return bpf_program__is_type(prog, TYPE); \
-} \
-
-BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
-BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM);
-BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
-BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
-BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
-BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
-BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
-BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
-BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
-BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
-BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
-BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT);
-BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP);
-
__alias(bpf_program__expected_attach_type)
enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
@@ -8991,6 +8433,7 @@ int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log
static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
+static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
@@ -9000,19 +8443,23 @@ static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_li
static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
static const struct bpf_sec_def section_defs[] = {
- SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
+ SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE),
+ SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE),
+ SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE),
SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
+ SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
+ SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
+ SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
+ SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE),
- SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE | SEC_SLOPPY_PFX | SEC_DEPRECATED),
- SEC_DEF("action", SCHED_ACT, 0, SEC_NONE | SEC_SLOPPY_PFX),
+ SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE),
+ SEC_DEF("action", SCHED_ACT, 0, SEC_NONE),
SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp),
SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp),
SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp),
@@ -9029,55 +8476,54 @@ static const struct bpf_sec_def section_defs[] = {
SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace),
SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm),
SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm),
+ SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF),
SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter),
SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter),
SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE),
SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS),
SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE),
- SEC_DEF("xdp_devmap/", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE | SEC_DEPRECATED),
SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS),
SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE),
- SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE | SEC_DEPRECATED),
SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS),
- SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE | SEC_SLOPPY_PFX),
- SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
- SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
+ SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT),
+ SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE),
+ SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE),
+ SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE),
+ SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE),
+ SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE),
+ SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
+ SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
+ SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
+ SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE),
+ SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
+ SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
+ SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT),
+ SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT),
+ SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT),
+ SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE),
+ SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT),
+ SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE),
+ SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT),
SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE),
- SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE | SEC_SLOPPY_PFX),
+ SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE),
};
static size_t custom_sec_def_cnt;
@@ -9172,8 +8618,7 @@ int libbpf_unregister_prog_handler(int handler_id)
return 0;
}
-static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name,
- bool allow_sloppy)
+static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name)
{
size_t len = strlen(sec_def->sec);
@@ -9198,17 +8643,6 @@ static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_n
return false;
}
- /* SEC_SLOPPY_PFX definitions are allowed to be just prefix
- * matches, unless strict section name mode
- * (LIBBPF_STRICT_SEC_NAME) is enabled, in which case the
- * match has to be exact.
- */
- if (allow_sloppy && str_has_pfx(sec_name, sec_def->sec))
- return true;
-
- /* Definitions not marked SEC_SLOPPY_PFX (e.g.,
- * SEC("syscall")) are exact matches in both modes.
- */
return strcmp(sec_name, sec_def->sec) == 0;
}
@@ -9216,20 +8650,18 @@ static const struct bpf_sec_def *find_sec_def(const char *sec_name)
{
const struct bpf_sec_def *sec_def;
int i, n;
- bool strict = libbpf_mode & LIBBPF_STRICT_SEC_NAME, allow_sloppy;
n = custom_sec_def_cnt;
for (i = 0; i < n; i++) {
sec_def = &custom_sec_defs[i];
- if (sec_def_matches(sec_def, sec_name, false))
+ if (sec_def_matches(sec_def, sec_name))
return sec_def;
}
n = ARRAY_SIZE(section_defs);
for (i = 0; i < n; i++) {
sec_def = &section_defs[i];
- allow_sloppy = (sec_def->cookie & SEC_SLOPPY_PFX) && !strict;
- if (sec_def_matches(sec_def, sec_name, allow_sloppy))
+ if (sec_def_matches(sec_def, sec_name))
return sec_def;
}
@@ -9300,6 +8732,38 @@ int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
return libbpf_err(-ESRCH);
}
+const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
+{
+ if (t < 0 || t >= ARRAY_SIZE(attach_type_name))
+ return NULL;
+
+ return attach_type_name[t];
+}
+
+const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
+{
+ if (t < 0 || t >= ARRAY_SIZE(link_type_name))
+ return NULL;
+
+ return link_type_name[t];
+}
+
+const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
+{
+ if (t < 0 || t >= ARRAY_SIZE(map_type_name))
+ return NULL;
+
+ return map_type_name[t];
+}
+
+const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
+{
+ if (t < 0 || t >= ARRAY_SIZE(prog_type_name))
+ return NULL;
+
+ return prog_type_name[t];
+}
+
static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
size_t offset)
{
@@ -9450,6 +8914,7 @@ void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,
*kind = BTF_KIND_TYPEDEF;
break;
case BPF_LSM_MAC:
+ case BPF_LSM_CGROUP:
*prefix = BTF_LSM_PREFIX;
*kind = BTF_KIND_FUNC;
break;
@@ -9653,11 +9118,6 @@ int bpf_map__fd(const struct bpf_map *map)
return map ? map->fd : libbpf_err(-EINVAL);
}
-const struct bpf_map_def *bpf_map__def(const struct bpf_map *map)
-{
- return map ? &map->def : libbpf_err_ptr(-EINVAL);
-}
-
static bool map_uses_real_name(const struct bpf_map *map)
{
/* Since libbpf started to support custom .data.* and .rodata.* maps,
@@ -9772,27 +9232,6 @@ __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
return map ? map->btf_value_type_id : 0;
}
-int bpf_map__set_priv(struct bpf_map *map, void *priv,
- bpf_map_clear_priv_t clear_priv)
-{
- if (!map)
- return libbpf_err(-EINVAL);
-
- if (map->priv) {
- if (map->clear_priv)
- map->clear_priv(map, map->priv);
- }
-
- map->priv = priv;
- map->clear_priv = clear_priv;
- return 0;
-}
-
-void *bpf_map__priv(const struct bpf_map *map)
-{
- return map ? map->priv : libbpf_err_ptr(-EINVAL);
-}
-
int bpf_map__set_initial_value(struct bpf_map *map,
const void *data, size_t size)
{
@@ -9812,11 +9251,6 @@ const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize)
return map->mmaped;
}
-bool bpf_map__is_offload_neutral(const struct bpf_map *map)
-{
- return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
-}
-
bool bpf_map__is_internal(const struct bpf_map *map)
{
return map->libbpf_type != LIBBPF_MAP_UNSPEC;
@@ -9878,12 +9312,6 @@ __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
}
struct bpf_map *
-bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj)
-{
- return bpf_object__next_map(obj, prev);
-}
-
-struct bpf_map *
bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
{
if (prev == NULL)
@@ -9893,12 +9321,6 @@ bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
}
struct bpf_map *
-bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj)
-{
- return bpf_object__prev_map(obj, next);
-}
-
-struct bpf_map *
bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
{
if (next == NULL) {
@@ -9943,12 +9365,6 @@ bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
}
-struct bpf_map *
-bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
-{
- return libbpf_err_ptr(-ENOTSUP);
-}
-
static int validate_map_op(const struct bpf_map *map, size_t key_sz,
size_t value_sz, bool check_value_sz)
{
@@ -10069,95 +9485,6 @@ long libbpf_get_error(const void *ptr)
return -errno;
}
-__attribute__((alias("bpf_prog_load_xattr2")))
-int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
- struct bpf_object **pobj, int *prog_fd);
-
-static int bpf_prog_load_xattr2(const struct bpf_prog_load_attr *attr,
- struct bpf_object **pobj, int *prog_fd)
-{
- struct bpf_object_open_attr open_attr = {};
- struct bpf_program *prog, *first_prog = NULL;
- struct bpf_object *obj;
- struct bpf_map *map;
- int err;
-
- if (!attr)
- return libbpf_err(-EINVAL);
- if (!attr->file)
- return libbpf_err(-EINVAL);
-
- open_attr.file = attr->file;
- open_attr.prog_type = attr->prog_type;
-
- obj = __bpf_object__open_xattr(&open_attr, 0);
- err = libbpf_get_error(obj);
- if (err)
- return libbpf_err(-ENOENT);
-
- bpf_object__for_each_program(prog, obj) {
- enum bpf_attach_type attach_type = attr->expected_attach_type;
- /*
- * to preserve backwards compatibility, bpf_prog_load treats
- * attr->prog_type, if specified, as an override to whatever
- * bpf_object__open guessed
- */
- if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
- prog->type = attr->prog_type;
- prog->expected_attach_type = attach_type;
- }
- if (bpf_program__type(prog) == BPF_PROG_TYPE_UNSPEC) {
- /*
- * we haven't guessed from section name and user
- * didn't provide a fallback type, too bad...
- */
- bpf_object__close(obj);
- return libbpf_err(-EINVAL);
- }
-
- prog->prog_ifindex = attr->ifindex;
- prog->log_level = attr->log_level;
- prog->prog_flags |= attr->prog_flags;
- if (!first_prog)
- first_prog = prog;
- }
-
- bpf_object__for_each_map(map, obj) {
- if (map->def.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
- map->map_ifindex = attr->ifindex;
- }
-
- if (!first_prog) {
- pr_warn("object file doesn't contain bpf program\n");
- bpf_object__close(obj);
- return libbpf_err(-ENOENT);
- }
-
- err = bpf_object__load(obj);
- if (err) {
- bpf_object__close(obj);
- return libbpf_err(err);
- }
-
- *pobj = obj;
- *prog_fd = bpf_program__fd(first_prog);
- return 0;
-}
-
-COMPAT_VERSION(bpf_prog_load_deprecated, bpf_prog_load, LIBBPF_0.0.1)
-int bpf_prog_load_deprecated(const char *file, enum bpf_prog_type type,
- struct bpf_object **pobj, int *prog_fd)
-{
- struct bpf_prog_load_attr attr;
-
- memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
- attr.file = file;
- attr.prog_type = type;
- attr.expected_attach_type = 0;
-
- return bpf_prog_load_xattr2(&attr, pobj, prog_fd);
-}
-
/* Replace link's underlying BPF program with the new one */
int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
{
@@ -10485,7 +9812,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
{
struct perf_event_attr attr = {};
char errmsg[STRERR_BUFSIZE];
- int type, pfd, err;
+ int type, pfd;
if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
return -EINVAL;
@@ -10521,14 +9848,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
pid < 0 ? -1 : pid /* pid */,
pid == -1 ? 0 : -1 /* cpu */,
-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
- if (pfd < 0) {
- err = -errno;
- pr_warn("%s perf_event_open() failed: %s\n",
- uprobe ? "uprobe" : "kprobe",
- libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
- return err;
- }
- return pfd;
+ return pfd >= 0 ? pfd : -errno;
}
static int append_to_file(const char *file, const char *fmt, ...)
@@ -10551,6 +9871,34 @@ static int append_to_file(const char *file, const char *fmt, ...)
return err;
}
+#define DEBUGFS "/sys/kernel/debug/tracing"
+#define TRACEFS "/sys/kernel/tracing"
+
+static bool use_debugfs(void)
+{
+ static int has_debugfs = -1;
+
+ if (has_debugfs < 0)
+ has_debugfs = access(DEBUGFS, F_OK) == 0;
+
+ return has_debugfs == 1;
+}
+
+static const char *tracefs_path(void)
+{
+ return use_debugfs() ? DEBUGFS : TRACEFS;
+}
+
+static const char *tracefs_kprobe_events(void)
+{
+ return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events";
+}
+
+static const char *tracefs_uprobe_events(void)
+{
+ return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events";
+}
+
static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
const char *kfunc_name, size_t offset)
{
@@ -10563,9 +9911,7 @@ static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz,
static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
const char *kfunc_name, size_t offset)
{
- const char *file = "/sys/kernel/debug/tracing/kprobe_events";
-
- return append_to_file(file, "%c:%s/%s %s+0x%zx",
+ return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx",
retprobe ? 'r' : 'p',
retprobe ? "kretprobes" : "kprobes",
probe_name, kfunc_name, offset);
@@ -10573,18 +9919,16 @@ static int add_kprobe_event_legacy(const char *probe_name, bool retprobe,
static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe)
{
- const char *file = "/sys/kernel/debug/tracing/kprobe_events";
-
- return append_to_file(file, "-:%s/%s", retprobe ? "kretprobes" : "kprobes", probe_name);
+ return append_to_file(tracefs_kprobe_events(), "-:%s/%s",
+ retprobe ? "kretprobes" : "kprobes", probe_name);
}
static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe)
{
char file[256];
- snprintf(file, sizeof(file),
- "/sys/kernel/debug/tracing/events/%s/%s/id",
- retprobe ? "kretprobes" : "kprobes", probe_name);
+ snprintf(file, sizeof(file), "%s/events/%s/%s/id",
+ tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name);
return parse_uint_from_file(file, "%d\n");
}
@@ -10605,10 +9949,11 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
}
type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
if (type < 0) {
+ err = type;
pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
kfunc_name, offset,
- libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
- return type;
+ libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
+ goto err_clean_legacy;
}
attr.size = sizeof(attr);
attr.config = type;
@@ -10622,9 +9967,72 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
err = -errno;
pr_warn("legacy kprobe perf_event_open() failed: %s\n",
libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
- return err;
+ goto err_clean_legacy;
}
return pfd;
+
+err_clean_legacy:
+ /* Clear the newly added legacy kprobe_event */
+ remove_kprobe_event_legacy(probe_name, retprobe);
+ return err;
+}
+
+static const char *arch_specific_syscall_pfx(void)
+{
+#if defined(__x86_64__)
+ return "x64";
+#elif defined(__i386__)
+ return "ia32";
+#elif defined(__s390x__)
+ return "s390x";
+#elif defined(__s390__)
+ return "s390";
+#elif defined(__arm__)
+ return "arm";
+#elif defined(__aarch64__)
+ return "arm64";
+#elif defined(__mips__)
+ return "mips";
+#elif defined(__riscv)
+ return "riscv";
+#elif defined(__powerpc__)
+ return "powerpc";
+#elif defined(__powerpc64__)
+ return "powerpc64";
+#else
+ return NULL;
+#endif
+}
+
+static int probe_kern_syscall_wrapper(void)
+{
+ char syscall_name[64];
+ const char *ksys_pfx;
+
+ ksys_pfx = arch_specific_syscall_pfx();
+ if (!ksys_pfx)
+ return 0;
+
+ snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx);
+
+ if (determine_kprobe_perf_type() >= 0) {
+ int pfd;
+
+ pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0);
+ if (pfd >= 0)
+ close(pfd);
+
+ return pfd >= 0 ? 1 : 0;
+ } else { /* legacy mode */
+ char probe_name[128];
+
+ gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0);
+ if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0)
+ return 0;
+
+ (void)remove_kprobe_event_legacy(probe_name, false);
+ return 1;
+ }
}
struct bpf_link *
@@ -10681,7 +10089,7 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
prog->name, retprobe ? "kretprobe" : "kprobe",
func_name, offset,
libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
- goto err_out;
+ goto err_clean_legacy;
}
if (legacy) {
struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
@@ -10692,6 +10100,10 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
}
return link;
+
+err_clean_legacy:
+ if (legacy)
+ remove_kprobe_event_legacy(legacy_probe, retprobe);
err_out:
free(legacy_probe);
return libbpf_err_ptr(err);
@@ -10708,6 +10120,34 @@ struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
}
+struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
+ const char *syscall_name,
+ const struct bpf_ksyscall_opts *opts)
+{
+ LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts);
+ char func_name[128];
+
+ if (!OPTS_VALID(opts, bpf_ksyscall_opts))
+ return libbpf_err_ptr(-EINVAL);
+
+ if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
+ /* arch_specific_syscall_pfx() should never return NULL here
+ * because it is guarded by kernel_supports(). However, since
+ * compiler does not know that we have an explicit conditional
+ * as well.
+ */
+ snprintf(func_name, sizeof(func_name), "__%s_sys_%s",
+ arch_specific_syscall_pfx() ? : "", syscall_name);
+ } else {
+ snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name);
+ }
+
+ kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false);
+ kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
+
+ return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
+}
+
/* Adapted from perf/util/string.c */
static bool glob_match(const char *str, const char *pat)
{
@@ -10878,6 +10318,27 @@ static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf
return libbpf_get_error(*link);
}
+static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
+{
+ LIBBPF_OPTS(bpf_ksyscall_opts, opts);
+ const char *syscall_name;
+
+ *link = NULL;
+
+ /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */
+ if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
+ return 0;
+
+ opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
+ if (opts.retprobe)
+ syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
+ else
+ syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
+
+ *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
+ return *link ? 0 : -errno;
+}
+
static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
{
LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
@@ -10926,9 +10387,7 @@ static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz,
static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
const char *binary_path, size_t offset)
{
- const char *file = "/sys/kernel/debug/tracing/uprobe_events";
-
- return append_to_file(file, "%c:%s/%s %s:0x%zx",
+ return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx",
retprobe ? 'r' : 'p',
retprobe ? "uretprobes" : "uprobes",
probe_name, binary_path, offset);
@@ -10936,18 +10395,16 @@ static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe,
static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe)
{
- const char *file = "/sys/kernel/debug/tracing/uprobe_events";
-
- return append_to_file(file, "-:%s/%s", retprobe ? "uretprobes" : "uprobes", probe_name);
+ return append_to_file(tracefs_uprobe_events(), "-:%s/%s",
+ retprobe ? "uretprobes" : "uprobes", probe_name);
}
static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe)
{
char file[512];
- snprintf(file, sizeof(file),
- "/sys/kernel/debug/tracing/events/%s/%s/id",
- retprobe ? "uretprobes" : "uprobes", probe_name);
+ snprintf(file, sizeof(file), "%s/events/%s/%s/id",
+ tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name);
return parse_uint_from_file(file, "%d\n");
}
@@ -10966,9 +10423,10 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
}
type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
if (type < 0) {
+ err = type;
pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",
binary_path, offset, err);
- return type;
+ goto err_clean_legacy;
}
memset(&attr, 0, sizeof(attr));
@@ -10983,46 +10441,14 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
if (pfd < 0) {
err = -errno;
pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);
- return err;
+ goto err_clean_legacy;
}
return pfd;
-}
-/* uprobes deal in relative offsets; subtract the base address associated with
- * the mapped binary. See Documentation/trace/uprobetracer.rst for more
- * details.
- */
-static long elf_find_relative_offset(const char *filename, Elf *elf, long addr)
-{
- size_t n;
- int i;
-
- if (elf_getphdrnum(elf, &n)) {
- pr_warn("elf: failed to find program headers for '%s': %s\n", filename,
- elf_errmsg(-1));
- return -ENOENT;
- }
-
- for (i = 0; i < n; i++) {
- int seg_start, seg_end, seg_offset;
- GElf_Phdr phdr;
-
- if (!gelf_getphdr(elf, i, &phdr)) {
- pr_warn("elf: failed to get program header %d from '%s': %s\n", i, filename,
- elf_errmsg(-1));
- return -ENOENT;
- }
- if (phdr.p_type != PT_LOAD || !(phdr.p_flags & PF_X))
- continue;
-
- seg_start = phdr.p_vaddr;
- seg_end = seg_start + phdr.p_memsz;
- seg_offset = phdr.p_offset;
- if (addr >= seg_start && addr < seg_end)
- return addr - seg_start + seg_offset;
- }
- pr_warn("elf: failed to find prog header containing 0x%lx in '%s'\n", addr, filename);
- return -ENOENT;
+err_clean_legacy:
+ /* Clear the newly added legacy uprobe_event */
+ remove_uprobe_event_legacy(probe_name, retprobe);
+ return err;
}
/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
@@ -11111,6 +10537,8 @@ static long elf_find_func_offset(const char *binary_path, const char *name)
for (idx = 0; idx < nr_syms; idx++) {
int curr_bind;
GElf_Sym sym;
+ Elf_Scn *sym_scn;
+ GElf_Shdr sym_sh;
if (!gelf_getsym(symbols, idx, &sym))
continue;
@@ -11148,12 +10576,28 @@ static long elf_find_func_offset(const char *binary_path, const char *name)
continue;
}
}
- ret = sym.st_value;
+
+ /* Transform symbol's virtual address (absolute for
+ * binaries and relative for shared libs) into file
+ * offset, which is what kernel is expecting for
+ * uprobe/uretprobe attachment.
+ * See Documentation/trace/uprobetracer.rst for more
+ * details.
+ * This is done by looking up symbol's containing
+ * section's header and using it's virtual address
+ * (sh_addr) and corresponding file offset (sh_offset)
+ * to transform sym.st_value (virtual address) into
+ * desired final file offset.
+ */
+ sym_scn = elf_getscn(elf, sym.st_shndx);
+ if (!sym_scn)
+ continue;
+ if (!gelf_getshdr(sym_scn, &sym_sh))
+ continue;
+
+ ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
last_bind = curr_bind;
}
- /* For binaries that are not shared libraries, we need relative offset */
- if (ret > 0 && !is_shared_lib)
- ret = elf_find_relative_offset(binary_path, elf, ret);
if (ret > 0)
break;
}
@@ -11276,7 +10720,10 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0);
pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
- if (binary_path && !strchr(binary_path, '/')) {
+ if (!binary_path)
+ return libbpf_err_ptr(-EINVAL);
+
+ if (!strchr(binary_path, '/')) {
err = resolve_full_path(binary_path, full_binary_path,
sizeof(full_binary_path));
if (err) {
@@ -11290,11 +10737,6 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
if (func_name) {
long sym_off;
- if (!binary_path) {
- pr_warn("prog '%s': name-based attach requires binary_path\n",
- prog->name);
- return libbpf_err_ptr(-EINVAL);
- }
sym_off = elf_find_func_offset(binary_path, func_name);
if (sym_off < 0)
return libbpf_err_ptr(sym_off);
@@ -11338,7 +10780,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
prog->name, retprobe ? "uretprobe" : "uprobe",
binary_path, func_offset,
libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
- goto err_out;
+ goto err_clean_legacy;
}
if (legacy) {
struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
@@ -11348,10 +10790,13 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
perf_link->legacy_is_retprobe = retprobe;
}
return link;
+
+err_clean_legacy:
+ if (legacy)
+ remove_uprobe_event_legacy(legacy_probe, retprobe);
err_out:
free(legacy_probe);
return libbpf_err_ptr(err);
-
}
/* Format of u[ret]probe section definition supporting auto-attach:
@@ -11386,7 +10831,8 @@ static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf
break;
case 3:
case 4:
- opts.retprobe = strcmp(probe_type, "uretprobe") == 0;
+ opts.retprobe = strcmp(probe_type, "uretprobe") == 0 ||
+ strcmp(probe_type, "uretprobe.s") == 0;
if (opts.retprobe && offset != 0) {
pr_warn("prog '%s': uretprobes do not support offset specification\n",
prog->name);
@@ -11438,6 +10884,9 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
return libbpf_err_ptr(-EINVAL);
}
+ if (!binary_path)
+ return libbpf_err_ptr(-EINVAL);
+
if (!strchr(binary_path, '/')) {
err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
if (err) {
@@ -11503,9 +10952,8 @@ static int determine_tracepoint_id(const char *tp_category,
char file[PATH_MAX];
int ret;
- ret = snprintf(file, sizeof(file),
- "/sys/kernel/debug/tracing/events/%s/%s/id",
- tp_category, tp_name);
+ ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id",
+ tracefs_path(), tp_category, tp_name);
if (ret < 0)
return -errno;
if (ret >= sizeof(file)) {
@@ -11962,6 +11410,9 @@ struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
return link;
}
+typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
+ void *private_data);
+
static enum bpf_perf_event_ret
perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
void **copy_mem, size_t *copy_size,
@@ -12010,12 +11461,6 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
return libbpf_err(ret);
}
-__attribute__((alias("perf_event_read_simple")))
-enum bpf_perf_event_ret
-bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
- void **copy_mem, size_t *copy_size,
- bpf_perf_event_print_t fn, void *private_data);
-
struct perf_buffer;
struct perf_buffer_params {
@@ -12149,12 +11594,11 @@ error:
static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
struct perf_buffer_params *p);
-DEFAULT_VERSION(perf_buffer__new_v0_6_0, perf_buffer__new, LIBBPF_0.6.0)
-struct perf_buffer *perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt,
- perf_buffer_sample_fn sample_cb,
- perf_buffer_lost_fn lost_cb,
- void *ctx,
- const struct perf_buffer_opts *opts)
+struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
+ perf_buffer_sample_fn sample_cb,
+ perf_buffer_lost_fn lost_cb,
+ void *ctx,
+ const struct perf_buffer_opts *opts)
{
struct perf_buffer_params p = {};
struct perf_event_attr attr = {};
@@ -12176,22 +11620,10 @@ struct perf_buffer *perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt,
return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
}
-COMPAT_VERSION(perf_buffer__new_deprecated, perf_buffer__new, LIBBPF_0.0.4)
-struct perf_buffer *perf_buffer__new_deprecated(int map_fd, size_t page_cnt,
- const struct perf_buffer_opts *opts)
-{
- return perf_buffer__new_v0_6_0(map_fd, page_cnt,
- opts ? opts->sample_cb : NULL,
- opts ? opts->lost_cb : NULL,
- opts ? opts->ctx : NULL,
- NULL);
-}
-
-DEFAULT_VERSION(perf_buffer__new_raw_v0_6_0, perf_buffer__new_raw, LIBBPF_0.6.0)
-struct perf_buffer *perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt,
- struct perf_event_attr *attr,
- perf_buffer_event_fn event_cb, void *ctx,
- const struct perf_buffer_raw_opts *opts)
+struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt,
+ struct perf_event_attr *attr,
+ perf_buffer_event_fn event_cb, void *ctx,
+ const struct perf_buffer_raw_opts *opts)
{
struct perf_buffer_params p = {};
@@ -12211,20 +11643,6 @@ struct perf_buffer *perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt,
return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p));
}
-COMPAT_VERSION(perf_buffer__new_raw_deprecated, perf_buffer__new_raw, LIBBPF_0.0.4)
-struct perf_buffer *perf_buffer__new_raw_deprecated(int map_fd, size_t page_cnt,
- const struct perf_buffer_raw_opts *opts)
-{
- LIBBPF_OPTS(perf_buffer_raw_opts, inner_opts,
- .cpu_cnt = opts->cpu_cnt,
- .cpus = opts->cpus,
- .map_keys = opts->map_keys,
- );
-
- return perf_buffer__new_raw_v0_6_0(map_fd, page_cnt, opts->attr,
- opts->event_cb, opts->ctx, &inner_opts);
-}
-
static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt,
struct perf_buffer_params *p)
{
@@ -12485,6 +11903,22 @@ int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx)
return cpu_buf->fd;
}
+int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size)
+{
+ struct perf_cpu_buf *cpu_buf;
+
+ if (buf_idx >= pb->cpu_cnt)
+ return libbpf_err(-EINVAL);
+
+ cpu_buf = pb->cpu_bufs[buf_idx];
+ if (!cpu_buf)
+ return libbpf_err(-ENOENT);
+
+ *buf = cpu_buf->base;
+ *buf_size = pb->mmap_size;
+ return 0;
+}
+
/*
* Consume data from perf ring buffer corresponding to slot *buf_idx* in
* PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to
@@ -12526,254 +11960,6 @@ int perf_buffer__consume(struct perf_buffer *pb)
return 0;
}
-struct bpf_prog_info_array_desc {
- int array_offset; /* e.g. offset of jited_prog_insns */
- int count_offset; /* e.g. offset of jited_prog_len */
- int size_offset; /* > 0: offset of rec size,
- * < 0: fix size of -size_offset
- */
-};
-
-static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
- [BPF_PROG_INFO_JITED_INSNS] = {
- offsetof(struct bpf_prog_info, jited_prog_insns),
- offsetof(struct bpf_prog_info, jited_prog_len),
- -1,
- },
- [BPF_PROG_INFO_XLATED_INSNS] = {
- offsetof(struct bpf_prog_info, xlated_prog_insns),
- offsetof(struct bpf_prog_info, xlated_prog_len),
- -1,
- },
- [BPF_PROG_INFO_MAP_IDS] = {
- offsetof(struct bpf_prog_info, map_ids),
- offsetof(struct bpf_prog_info, nr_map_ids),
- -(int)sizeof(__u32),
- },
- [BPF_PROG_INFO_JITED_KSYMS] = {
- offsetof(struct bpf_prog_info, jited_ksyms),
- offsetof(struct bpf_prog_info, nr_jited_ksyms),
- -(int)sizeof(__u64),
- },
- [BPF_PROG_INFO_JITED_FUNC_LENS] = {
- offsetof(struct bpf_prog_info, jited_func_lens),
- offsetof(struct bpf_prog_info, nr_jited_func_lens),
- -(int)sizeof(__u32),
- },
- [BPF_PROG_INFO_FUNC_INFO] = {
- offsetof(struct bpf_prog_info, func_info),
- offsetof(struct bpf_prog_info, nr_func_info),
- offsetof(struct bpf_prog_info, func_info_rec_size),
- },
- [BPF_PROG_INFO_LINE_INFO] = {
- offsetof(struct bpf_prog_info, line_info),
- offsetof(struct bpf_prog_info, nr_line_info),
- offsetof(struct bpf_prog_info, line_info_rec_size),
- },
- [BPF_PROG_INFO_JITED_LINE_INFO] = {
- offsetof(struct bpf_prog_info, jited_line_info),
- offsetof(struct bpf_prog_info, nr_jited_line_info),
- offsetof(struct bpf_prog_info, jited_line_info_rec_size),
- },
- [BPF_PROG_INFO_PROG_TAGS] = {
- offsetof(struct bpf_prog_info, prog_tags),
- offsetof(struct bpf_prog_info, nr_prog_tags),
- -(int)sizeof(__u8) * BPF_TAG_SIZE,
- },
-
-};
-
-static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info,
- int offset)
-{
- __u32 *array = (__u32 *)info;
-
- if (offset >= 0)
- return array[offset / sizeof(__u32)];
- return -(int)offset;
-}
-
-static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info,
- int offset)
-{
- __u64 *array = (__u64 *)info;
-
- if (offset >= 0)
- return array[offset / sizeof(__u64)];
- return -(int)offset;
-}
-
-static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
- __u32 val)
-{
- __u32 *array = (__u32 *)info;
-
- if (offset >= 0)
- array[offset / sizeof(__u32)] = val;
-}
-
-static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
- __u64 val)
-{
- __u64 *array = (__u64 *)info;
-
- if (offset >= 0)
- array[offset / sizeof(__u64)] = val;
-}
-
-struct bpf_prog_info_linear *
-bpf_program__get_prog_info_linear(int fd, __u64 arrays)
-{
- struct bpf_prog_info_linear *info_linear;
- struct bpf_prog_info info = {};
- __u32 info_len = sizeof(info);
- __u32 data_len = 0;
- int i, err;
- void *ptr;
-
- if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
- return libbpf_err_ptr(-EINVAL);
-
- /* step 1: get array dimensions */
- err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
- if (err) {
- pr_debug("can't get prog info: %s", strerror(errno));
- return libbpf_err_ptr(-EFAULT);
- }
-
- /* step 2: calculate total size of all arrays */
- for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
- bool include_array = (arrays & (1UL << i)) > 0;
- struct bpf_prog_info_array_desc *desc;
- __u32 count, size;
-
- desc = bpf_prog_info_array_desc + i;
-
- /* kernel is too old to support this field */
- if (info_len < desc->array_offset + sizeof(__u32) ||
- info_len < desc->count_offset + sizeof(__u32) ||
- (desc->size_offset > 0 && info_len < desc->size_offset))
- include_array = false;
-
- if (!include_array) {
- arrays &= ~(1UL << i); /* clear the bit */
- continue;
- }
-
- count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
- size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
-
- data_len += count * size;
- }
-
- /* step 3: allocate continuous memory */
- data_len = roundup(data_len, sizeof(__u64));
- info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
- if (!info_linear)
- return libbpf_err_ptr(-ENOMEM);
-
- /* step 4: fill data to info_linear->info */
- info_linear->arrays = arrays;
- memset(&info_linear->info, 0, sizeof(info));
- ptr = info_linear->data;
-
- for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
- struct bpf_prog_info_array_desc *desc;
- __u32 count, size;
-
- if ((arrays & (1UL << i)) == 0)
- continue;
-
- desc = bpf_prog_info_array_desc + i;
- count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
- size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
- bpf_prog_info_set_offset_u32(&info_linear->info,
- desc->count_offset, count);
- bpf_prog_info_set_offset_u32(&info_linear->info,
- desc->size_offset, size);
- bpf_prog_info_set_offset_u64(&info_linear->info,
- desc->array_offset,
- ptr_to_u64(ptr));
- ptr += count * size;
- }
-
- /* step 5: call syscall again to get required arrays */
- err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
- if (err) {
- pr_debug("can't get prog info: %s", strerror(errno));
- free(info_linear);
- return libbpf_err_ptr(-EFAULT);
- }
-
- /* step 6: verify the data */
- for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
- struct bpf_prog_info_array_desc *desc;
- __u32 v1, v2;
-
- if ((arrays & (1UL << i)) == 0)
- continue;
-
- desc = bpf_prog_info_array_desc + i;
- v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
- v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
- desc->count_offset);
- if (v1 != v2)
- pr_warn("%s: mismatch in element count\n", __func__);
-
- v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
- v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
- desc->size_offset);
- if (v1 != v2)
- pr_warn("%s: mismatch in rec size\n", __func__);
- }
-
- /* step 7: update info_len and data_len */
- info_linear->info_len = sizeof(struct bpf_prog_info);
- info_linear->data_len = data_len;
-
- return info_linear;
-}
-
-void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
-{
- int i;
-
- for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
- struct bpf_prog_info_array_desc *desc;
- __u64 addr, offs;
-
- if ((info_linear->arrays & (1UL << i)) == 0)
- continue;
-
- desc = bpf_prog_info_array_desc + i;
- addr = bpf_prog_info_read_offset_u64(&info_linear->info,
- desc->array_offset);
- offs = addr - ptr_to_u64(info_linear->data);
- bpf_prog_info_set_offset_u64(&info_linear->info,
- desc->array_offset, offs);
- }
-}
-
-void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
-{
- int i;
-
- for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
- struct bpf_prog_info_array_desc *desc;
- __u64 addr, offs;
-
- if ((info_linear->arrays & (1UL << i)) == 0)
- continue;
-
- desc = bpf_prog_info_array_desc + i;
- offs = bpf_prog_info_read_offset_u64(&info_linear->info,
- desc->array_offset);
- addr = offs + ptr_to_u64(info_linear->data);
- bpf_prog_info_set_offset_u64(&info_linear->info,
- desc->array_offset, addr);
- }
-}
-
int bpf_program__set_attach_target(struct bpf_program *prog,
int attach_prog_fd,
const char *attach_func_name)
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 9e9a3fd3edd8..61493c4cddac 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -51,6 +51,42 @@ enum libbpf_errno {
LIBBPF_API int libbpf_strerror(int err, char *buf, size_t size);
+/**
+ * @brief **libbpf_bpf_attach_type_str()** converts the provided attach type
+ * value into a textual representation.
+ * @param t The attach type.
+ * @return Pointer to a static string identifying the attach type. NULL is
+ * returned for unknown **bpf_attach_type** values.
+ */
+LIBBPF_API const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t);
+
+/**
+ * @brief **libbpf_bpf_link_type_str()** converts the provided link type value
+ * into a textual representation.
+ * @param t The link type.
+ * @return Pointer to a static string identifying the link type. NULL is
+ * returned for unknown **bpf_link_type** values.
+ */
+LIBBPF_API const char *libbpf_bpf_link_type_str(enum bpf_link_type t);
+
+/**
+ * @brief **libbpf_bpf_map_type_str()** converts the provided map type value
+ * into a textual representation.
+ * @param t The map type.
+ * @return Pointer to a static string identifying the map type. NULL is
+ * returned for unknown **bpf_map_type** values.
+ */
+LIBBPF_API const char *libbpf_bpf_map_type_str(enum bpf_map_type t);
+
+/**
+ * @brief **libbpf_bpf_prog_type_str()** converts the provided program type
+ * value into a textual representation.
+ * @param t The program type.
+ * @return Pointer to a static string identifying the program type. NULL is
+ * returned for unknown **bpf_prog_type** values.
+ */
+LIBBPF_API const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t);
+
enum libbpf_print_level {
LIBBPF_WARN,
LIBBPF_INFO,
@@ -65,13 +101,8 @@ LIBBPF_API libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn);
/* Hide internal to user */
struct bpf_object;
-struct bpf_object_open_attr {
- const char *file;
- enum bpf_prog_type prog_type;
-};
-
struct bpf_object_open_opts {
- /* size of this struct, for forward/backward compatiblity */
+ /* size of this struct, for forward/backward compatibility */
size_t sz;
/* object name override, if provided:
* - for object open from file, this will override setting object
@@ -82,21 +113,12 @@ struct bpf_object_open_opts {
const char *object_name;
/* parse map definitions non-strictly, allowing extra attributes/data */
bool relaxed_maps;
- /* DEPRECATED: handle CO-RE relocations non-strictly, allowing failures.
- * Value is ignored. Relocations always are processed non-strictly.
- * Non-relocatable instructions are replaced with invalid ones to
- * prevent accidental errors.
- * */
- LIBBPF_DEPRECATED_SINCE(0, 6, "field has no effect")
- bool relaxed_core_relocs;
/* maps that set the 'pinning' attribute in their definition will have
* their pin_path attribute set to a file in this directory, and be
* auto-pinned to that path on load; defaults to "/sys/fs/bpf".
*/
const char *pin_root_path;
-
- LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__set_attach_target() on each individual bpf_program")
- __u32 attach_prog_fd;
+ long :0;
/* Additional kernel config content that augments and overrides
* system Kconfig for CONFIG_xxx externs.
*/
@@ -179,20 +201,10 @@ LIBBPF_API struct bpf_object *
bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
const struct bpf_object_open_opts *opts);
-/* deprecated bpf_object__open variants */
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__open_mem() instead")
-LIBBPF_API struct bpf_object *
-bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
- const char *name);
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__open_file() instead")
-LIBBPF_API struct bpf_object *
-bpf_object__open_xattr(struct bpf_object_open_attr *attr);
+/* Load/unload object into/from kernel */
+LIBBPF_API int bpf_object__load(struct bpf_object *obj);
-enum libbpf_pin_type {
- LIBBPF_PIN_NONE,
- /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
- LIBBPF_PIN_BY_NAME,
-};
+LIBBPF_API void bpf_object__close(struct bpf_object *object);
/* pin_maps and unpin_maps can both be called with a NULL path, in which case
* they will use the pin_path attribute of each map (and ignore all maps that
@@ -206,20 +218,6 @@ LIBBPF_API int bpf_object__pin_programs(struct bpf_object *obj,
LIBBPF_API int bpf_object__unpin_programs(struct bpf_object *obj,
const char *path);
LIBBPF_API int bpf_object__pin(struct bpf_object *object, const char *path);
-LIBBPF_API void bpf_object__close(struct bpf_object *object);
-
-struct bpf_object_load_attr {
- struct bpf_object *obj;
- int log_level;
- const char *target_btf_path;
-};
-
-/* Load/unload object into/from kernel */
-LIBBPF_API int bpf_object__load(struct bpf_object *obj);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__load() instead")
-LIBBPF_API int bpf_object__load_xattr(struct bpf_object_load_attr *attr);
-LIBBPF_DEPRECATED_SINCE(0, 6, "bpf_object__unload() is deprecated, use bpf_object__close() instead")
-LIBBPF_API int bpf_object__unload(struct bpf_object *obj);
LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj);
LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj);
@@ -229,29 +227,10 @@ struct btf;
LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj);
LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj);
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__find_program_by_name() instead")
-LIBBPF_API struct bpf_program *
-bpf_object__find_program_by_title(const struct bpf_object *obj,
- const char *title);
LIBBPF_API struct bpf_program *
bpf_object__find_program_by_name(const struct bpf_object *obj,
const char *name);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "track bpf_objects in application code instead")
-struct bpf_object *bpf_object__next(struct bpf_object *prev);
-#define bpf_object__for_each_safe(pos, tmp) \
- for ((pos) = bpf_object__next(NULL), \
- (tmp) = bpf_object__next(pos); \
- (pos) != NULL; \
- (pos) = (tmp), (tmp) = bpf_object__next(tmp))
-
-typedef void (*bpf_object_clear_priv_t)(struct bpf_object *, void *);
-LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated")
-LIBBPF_API int bpf_object__set_priv(struct bpf_object *obj, void *priv,
- bpf_object_clear_priv_t clear_priv);
-LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated")
-LIBBPF_API void *bpf_object__priv(const struct bpf_object *prog);
-
LIBBPF_API int
libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
enum bpf_attach_type *expected_attach_type);
@@ -262,9 +241,7 @@ LIBBPF_API int libbpf_find_vmlinux_btf_id(const char *name,
/* Accessors of bpf_program */
struct bpf_program;
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__next_program() instead")
-struct bpf_program *bpf_program__next(struct bpf_program *prog,
- const struct bpf_object *obj);
+
LIBBPF_API struct bpf_program *
bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prog);
@@ -273,33 +250,17 @@ bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prog)
(pos) != NULL; \
(pos) = bpf_object__next_program((obj), (pos)))
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__prev_program() instead")
-struct bpf_program *bpf_program__prev(struct bpf_program *prog,
- const struct bpf_object *obj);
LIBBPF_API struct bpf_program *
bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *prog);
-typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, void *);
-
-LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated")
-LIBBPF_API int bpf_program__set_priv(struct bpf_program *prog, void *priv,
- bpf_program_clear_priv_t clear_priv);
-LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated")
-LIBBPF_API void *bpf_program__priv(const struct bpf_program *prog);
LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog,
__u32 ifindex);
LIBBPF_API const char *bpf_program__name(const struct bpf_program *prog);
LIBBPF_API const char *bpf_program__section_name(const struct bpf_program *prog);
-LIBBPF_API LIBBPF_DEPRECATED("BPF program title is confusing term; please use bpf_program__section_name() instead")
-const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy);
LIBBPF_API bool bpf_program__autoload(const struct bpf_program *prog);
LIBBPF_API int bpf_program__set_autoload(struct bpf_program *prog, bool autoload);
-/* returns program size in bytes */
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__insn_cnt() instead")
-LIBBPF_API size_t bpf_program__size(const struct bpf_program *prog);
-
struct bpf_insn;
/**
@@ -352,17 +313,7 @@ LIBBPF_API int bpf_program__set_insns(struct bpf_program *prog,
*/
LIBBPF_API size_t bpf_program__insn_cnt(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 6, "use bpf_object__load() instead")
-LIBBPF_API int bpf_program__load(struct bpf_program *prog, const char *license, __u32 kern_version);
LIBBPF_API int bpf_program__fd(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated")
-LIBBPF_API int bpf_program__pin_instance(struct bpf_program *prog,
- const char *path,
- int instance);
-LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated")
-LIBBPF_API int bpf_program__unpin_instance(struct bpf_program *prog,
- const char *path,
- int instance);
/**
* @brief **bpf_program__pin()** pins the BPF program to a file
@@ -506,6 +457,52 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
const char *pattern,
const struct bpf_kprobe_multi_opts *opts);
+struct bpf_ksyscall_opts {
+ /* size of this struct, for forward/backward compatiblity */
+ size_t sz;
+ /* custom user-provided value fetchable through bpf_get_attach_cookie() */
+ __u64 bpf_cookie;
+ /* attach as return probe? */
+ bool retprobe;
+ size_t :0;
+};
+#define bpf_ksyscall_opts__last_field retprobe
+
+/**
+ * @brief **bpf_program__attach_ksyscall()** attaches a BPF program
+ * to kernel syscall handler of a specified syscall. Optionally it's possible
+ * to request to install retprobe that will be triggered at syscall exit. It's
+ * also possible to associate BPF cookie (though options).
+ *
+ * Libbpf automatically will determine correct full kernel function name,
+ * which depending on system architecture and kernel version/configuration
+ * could be of the form __<arch>_sys_<syscall> or __se_sys_<syscall>, and will
+ * attach specified program using kprobe/kretprobe mechanism.
+ *
+ * **bpf_program__attach_ksyscall()** is an API counterpart of declarative
+ * **SEC("ksyscall/<syscall>")** annotation of BPF programs.
+ *
+ * At the moment **SEC("ksyscall")** and **bpf_program__attach_ksyscall()** do
+ * not handle all the calling convention quirks for mmap(), clone() and compat
+ * syscalls. It also only attaches to "native" syscall interfaces. If host
+ * system supports compat syscalls or defines 32-bit syscalls in 64-bit
+ * kernel, such syscall interfaces won't be attached to by libbpf.
+ *
+ * These limitations may or may not change in the future. Therefore it is
+ * recommended to use SEC("kprobe") for these syscalls or if working with
+ * compat and 32-bit interfaces is required.
+ *
+ * @param prog BPF program to attach
+ * @param syscall_name Symbolic name of the syscall (e.g., "bpf")
+ * @param opts Additional options (see **struct bpf_ksyscall_opts**)
+ * @return Reference to the newly created BPF link; or NULL is returned on
+ * error, error code is stored in errno
+ */
+LIBBPF_API struct bpf_link *
+bpf_program__attach_ksyscall(const struct bpf_program *prog,
+ const char *syscall_name,
+ const struct bpf_ksyscall_opts *opts);
+
struct bpf_uprobe_opts {
/* size of this struct, for forward/backward compatiblity */
size_t sz;
@@ -662,99 +659,6 @@ LIBBPF_API struct bpf_link *
bpf_program__attach_iter(const struct bpf_program *prog,
const struct bpf_iter_attach_opts *opts);
-/*
- * Libbpf allows callers to adjust BPF programs before being loaded
- * into kernel. One program in an object file can be transformed into
- * multiple variants to be attached to different hooks.
- *
- * bpf_program_prep_t, bpf_program__set_prep and bpf_program__nth_fd
- * form an API for this purpose.
- *
- * - bpf_program_prep_t:
- * Defines a 'preprocessor', which is a caller defined function
- * passed to libbpf through bpf_program__set_prep(), and will be
- * called before program is loaded. The processor should adjust
- * the program one time for each instance according to the instance id
- * passed to it.
- *
- * - bpf_program__set_prep:
- * Attaches a preprocessor to a BPF program. The number of instances
- * that should be created is also passed through this function.
- *
- * - bpf_program__nth_fd:
- * After the program is loaded, get resulting FD of a given instance
- * of the BPF program.
- *
- * If bpf_program__set_prep() is not used, the program would be loaded
- * without adjustment during bpf_object__load(). The program has only
- * one instance. In this case bpf_program__fd(prog) is equal to
- * bpf_program__nth_fd(prog, 0).
- */
-struct bpf_prog_prep_result {
- /*
- * If not NULL, load new instruction array.
- * If set to NULL, don't load this instance.
- */
- struct bpf_insn *new_insn_ptr;
- int new_insn_cnt;
-
- /* If not NULL, result FD is written to it. */
- int *pfd;
-};
-
-/*
- * Parameters of bpf_program_prep_t:
- * - prog: The bpf_program being loaded.
- * - n: Index of instance being generated.
- * - insns: BPF instructions array.
- * - insns_cnt:Number of instructions in insns.
- * - res: Output parameter, result of transformation.
- *
- * Return value:
- * - Zero: pre-processing success.
- * - Non-zero: pre-processing error, stop loading.
- */
-typedef int (*bpf_program_prep_t)(struct bpf_program *prog, int n,
- struct bpf_insn *insns, int insns_cnt,
- struct bpf_prog_prep_result *res);
-
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__insns() for getting bpf_program instructions")
-LIBBPF_API int bpf_program__set_prep(struct bpf_program *prog, int nr_instance,
- bpf_program_prep_t prep);
-
-LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated")
-LIBBPF_API int bpf_program__nth_fd(const struct bpf_program *prog, int n);
-
-/*
- * Adjust type of BPF program. Default is kprobe.
- */
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_socket_filter(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_tracepoint(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_raw_tracepoint(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_kprobe(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_lsm(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_sched_cls(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_extension(struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
-LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog);
-
LIBBPF_API enum bpf_prog_type bpf_program__type(const struct bpf_program *prog);
/**
@@ -817,47 +721,6 @@ LIBBPF_API int
bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd,
const char *attach_func_name);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_socket_filter(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_tracepoint(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_raw_tracepoint(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_kprobe(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_lsm(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_struct_ops(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_extension(const struct bpf_program *prog);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead")
-LIBBPF_API bool bpf_program__is_sk_lookup(const struct bpf_program *prog);
-
-/*
- * No need for __attribute__((packed)), all members of 'bpf_map_def'
- * are all aligned. In addition, using __attribute__((packed))
- * would trigger a -Wpacked warning message, and lead to an error
- * if -Werror is set.
- */
-struct bpf_map_def {
- unsigned int type;
- unsigned int key_size;
- unsigned int value_size;
- unsigned int max_entries;
- unsigned int map_flags;
-};
-
/**
* @brief **bpf_object__find_map_by_name()** returns BPF map of
* the given name, if it exists within the passed BPF object
@@ -872,16 +735,6 @@ bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name);
LIBBPF_API int
bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name);
-/*
- * Get bpf_map through the offset of corresponding struct bpf_map_def
- * in the BPF object file.
- */
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__find_map_by_name() instead")
-struct bpf_map *
-bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset);
-
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__next_map() instead")
-struct bpf_map *bpf_map__next(const struct bpf_map *map, const struct bpf_object *obj);
LIBBPF_API struct bpf_map *
bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *map);
@@ -891,8 +744,6 @@ bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *map);
(pos) = bpf_object__next_map((obj), (pos)))
#define bpf_map__for_each bpf_object__for_each_map
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__prev_map() instead")
-struct bpf_map *bpf_map__prev(const struct bpf_map *map, const struct bpf_object *obj);
LIBBPF_API struct bpf_map *
bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *map);
@@ -926,9 +777,6 @@ LIBBPF_API bool bpf_map__autocreate(const struct bpf_map *map);
*/
LIBBPF_API int bpf_map__fd(const struct bpf_map *map);
LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd);
-/* get map definition */
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 8, "use appropriate getters or setters instead")
-const struct bpf_map_def *bpf_map__def(const struct bpf_map *map);
/* get map name */
LIBBPF_API const char *bpf_map__name(const struct bpf_map *map);
/* get/set map type */
@@ -937,8 +785,6 @@ LIBBPF_API int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type);
/* get/set map size (max_entries) */
LIBBPF_API __u32 bpf_map__max_entries(const struct bpf_map *map);
LIBBPF_API int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_map__set_max_entries() instead")
-LIBBPF_API int bpf_map__resize(struct bpf_map *map, __u32 max_entries);
/* get/set map flags */
LIBBPF_API __u32 bpf_map__map_flags(const struct bpf_map *map);
LIBBPF_API int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags);
@@ -961,17 +807,9 @@ LIBBPF_API int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex);
LIBBPF_API __u64 bpf_map__map_extra(const struct bpf_map *map);
LIBBPF_API int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra);
-typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *);
-LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated")
-LIBBPF_API int bpf_map__set_priv(struct bpf_map *map, void *priv,
- bpf_map_clear_priv_t clear_priv);
-LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated")
-LIBBPF_API void *bpf_map__priv(const struct bpf_map *map);
LIBBPF_API int bpf_map__set_initial_value(struct bpf_map *map,
const void *data, size_t size);
LIBBPF_API const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_map__type() instead")
-LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map);
/**
* @brief **bpf_map__is_internal()** tells the caller whether or not the
@@ -1094,65 +932,6 @@ LIBBPF_API int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
LIBBPF_API int bpf_map__get_next_key(const struct bpf_map *map,
const void *cur_key, void *next_key, size_t key_sz);
-/**
- * @brief **libbpf_get_error()** extracts the error code from the passed
- * pointer
- * @param ptr pointer returned from libbpf API function
- * @return error code; or 0 if no error occured
- *
- * Many libbpf API functions which return pointers have logic to encode error
- * codes as pointers, and do not return NULL. Meaning **libbpf_get_error()**
- * should be used on the return value from these functions immediately after
- * calling the API function, with no intervening calls that could clobber the
- * `errno` variable. Consult the individual functions documentation to verify
- * if this logic applies should be used.
- *
- * For these API functions, if `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)`
- * is enabled, NULL is returned on error instead.
- *
- * If ptr is NULL, then errno should be already set by the failing
- * API, because libbpf never returns NULL on success and it now always
- * sets errno on error.
- *
- * Example usage:
- *
- * struct perf_buffer *pb;
- *
- * pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, &opts);
- * err = libbpf_get_error(pb);
- * if (err) {
- * pb = NULL;
- * fprintf(stderr, "failed to open perf buffer: %d\n", err);
- * goto cleanup;
- * }
- */
-LIBBPF_API long libbpf_get_error(const void *ptr);
-
-struct bpf_prog_load_attr {
- const char *file;
- enum bpf_prog_type prog_type;
- enum bpf_attach_type expected_attach_type;
- int ifindex;
- int log_level;
- int prog_flags;
-};
-
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__open() and bpf_object__load() instead")
-LIBBPF_API int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
- struct bpf_object **pobj, int *prog_fd);
-LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__open() and bpf_object__load() instead")
-LIBBPF_API int bpf_prog_load_deprecated(const char *file, enum bpf_prog_type type,
- struct bpf_object **pobj, int *prog_fd);
-
-/* XDP related API */
-struct xdp_link_info {
- __u32 prog_id;
- __u32 drv_prog_id;
- __u32 hw_prog_id;
- __u32 skb_prog_id;
- __u8 attach_mode;
-};
-
struct bpf_xdp_set_link_opts {
size_t sz;
int old_fd;
@@ -1160,17 +939,6 @@ struct bpf_xdp_set_link_opts {
};
#define bpf_xdp_set_link_opts__last_field old_fd
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_attach() instead")
-LIBBPF_API int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_attach() instead")
-LIBBPF_API int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags,
- const struct bpf_xdp_set_link_opts *opts);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_query_id() instead")
-LIBBPF_API int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_query() instead")
-LIBBPF_API int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info,
- size_t info_size, __u32 flags);
-
struct bpf_xdp_attach_opts {
size_t sz;
int old_prog_fd;
@@ -1269,17 +1037,7 @@ typedef void (*perf_buffer_lost_fn)(void *ctx, int cpu, __u64 cnt);
/* common use perf buffer options */
struct perf_buffer_opts {
- union {
- size_t sz;
- struct { /* DEPRECATED: will be removed in v1.0 */
- /* if specified, sample_cb is called for each sample */
- perf_buffer_sample_fn sample_cb;
- /* if specified, lost_cb is called for each batch of lost samples */
- perf_buffer_lost_fn lost_cb;
- /* ctx is provided to sample_cb and lost_cb */
- void *ctx;
- };
- };
+ size_t sz;
};
#define perf_buffer_opts__last_field sz
@@ -1300,21 +1058,6 @@ perf_buffer__new(int map_fd, size_t page_cnt,
perf_buffer_sample_fn sample_cb, perf_buffer_lost_fn lost_cb, void *ctx,
const struct perf_buffer_opts *opts);
-LIBBPF_API struct perf_buffer *
-perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt,
- perf_buffer_sample_fn sample_cb, perf_buffer_lost_fn lost_cb, void *ctx,
- const struct perf_buffer_opts *opts);
-
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use new variant of perf_buffer__new() instead")
-struct perf_buffer *perf_buffer__new_deprecated(int map_fd, size_t page_cnt,
- const struct perf_buffer_opts *opts);
-
-#define perf_buffer__new(...) ___libbpf_overload(___perf_buffer_new, __VA_ARGS__)
-#define ___perf_buffer_new6(map_fd, page_cnt, sample_cb, lost_cb, ctx, opts) \
- perf_buffer__new(map_fd, page_cnt, sample_cb, lost_cb, ctx, opts)
-#define ___perf_buffer_new3(map_fd, page_cnt, opts) \
- perf_buffer__new_deprecated(map_fd, page_cnt, opts)
-
enum bpf_perf_event_ret {
LIBBPF_PERF_EVENT_DONE = 0,
LIBBPF_PERF_EVENT_ERROR = -1,
@@ -1328,21 +1071,9 @@ typedef enum bpf_perf_event_ret
/* raw perf buffer options, giving most power and control */
struct perf_buffer_raw_opts {
- union {
- struct {
- size_t sz;
- long :0;
- long :0;
- };
- struct { /* DEPRECATED: will be removed in v1.0 */
- /* perf event attrs passed directly into perf_event_open() */
- struct perf_event_attr *attr;
- /* raw event callback */
- perf_buffer_event_fn event_cb;
- /* ctx is provided to event_cb */
- void *ctx;
- };
- };
+ size_t sz;
+ long :0;
+ long :0;
/* if cpu_cnt == 0, open all on all possible CPUs (up to the number of
* max_entries of given PERF_EVENT_ARRAY map)
*/
@@ -1354,26 +1085,13 @@ struct perf_buffer_raw_opts {
};
#define perf_buffer_raw_opts__last_field map_keys
+struct perf_event_attr;
+
LIBBPF_API struct perf_buffer *
perf_buffer__new_raw(int map_fd, size_t page_cnt, struct perf_event_attr *attr,
perf_buffer_event_fn event_cb, void *ctx,
const struct perf_buffer_raw_opts *opts);
-LIBBPF_API struct perf_buffer *
-perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt, struct perf_event_attr *attr,
- perf_buffer_event_fn event_cb, void *ctx,
- const struct perf_buffer_raw_opts *opts);
-
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use new variant of perf_buffer__new_raw() instead")
-struct perf_buffer *perf_buffer__new_raw_deprecated(int map_fd, size_t page_cnt,
- const struct perf_buffer_raw_opts *opts);
-
-#define perf_buffer__new_raw(...) ___libbpf_overload(___perf_buffer_new_raw, __VA_ARGS__)
-#define ___perf_buffer_new_raw6(map_fd, page_cnt, attr, event_cb, ctx, opts) \
- perf_buffer__new_raw(map_fd, page_cnt, attr, event_cb, ctx, opts)
-#define ___perf_buffer_new_raw3(map_fd, page_cnt, opts) \
- perf_buffer__new_raw_deprecated(map_fd, page_cnt, opts)
-
LIBBPF_API void perf_buffer__free(struct perf_buffer *pb);
LIBBPF_API int perf_buffer__epoll_fd(const struct perf_buffer *pb);
LIBBPF_API int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms);
@@ -1381,15 +1099,22 @@ LIBBPF_API int perf_buffer__consume(struct perf_buffer *pb);
LIBBPF_API int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx);
LIBBPF_API size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb);
LIBBPF_API int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx);
-
-typedef enum bpf_perf_event_ret
- (*bpf_perf_event_print_t)(struct perf_event_header *hdr,
- void *private_data);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use perf_buffer__poll() or perf_buffer__consume() instead")
-LIBBPF_API enum bpf_perf_event_ret
-bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
- void **copy_mem, size_t *copy_size,
- bpf_perf_event_print_t fn, void *private_data);
+/**
+ * @brief **perf_buffer__buffer()** returns the per-cpu raw mmap()'ed underlying
+ * memory region of the ring buffer.
+ * This ring buffer can be used to implement a custom events consumer.
+ * The ring buffer starts with the *struct perf_event_mmap_page*, which
+ * holds the ring buffer managment fields, when accessing the header
+ * structure it's important to be SMP aware.
+ * You can refer to *perf_event_read_simple* for a simple example.
+ * @param pb the perf buffer structure
+ * @param buf_idx the buffer index to retreive
+ * @param buf (out) gets the base pointer of the mmap()'ed memory
+ * @param buf_size (out) gets the size of the mmap()'ed region
+ * @return 0 on success, negative error code for failure
+ */
+LIBBPF_API int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf,
+ size_t *buf_size);
struct bpf_prog_linfo;
struct bpf_prog_info;
@@ -1412,14 +1137,6 @@ bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo,
* user, causing subsequent probes to fail. In this case, the caller may want
* to adjust that limit with setrlimit().
*/
-LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_prog_type() instead")
-LIBBPF_API bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_map_type() instead")
-LIBBPF_API bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex);
-LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_helper() instead")
-LIBBPF_API bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, __u32 ifindex);
-LIBBPF_DEPRECATED_SINCE(0, 8, "implement your own or use bpftool for feature detection")
-LIBBPF_API bool bpf_probe_large_insn_limit(__u32 ifindex);
/**
* @brief **libbpf_probe_bpf_prog_type()** detects if host kernel supports
@@ -1463,72 +1180,6 @@ LIBBPF_API int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void
LIBBPF_API int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type,
enum bpf_func_id helper_id, const void *opts);
-/*
- * Get bpf_prog_info in continuous memory
- *
- * struct bpf_prog_info has multiple arrays. The user has option to choose
- * arrays to fetch from kernel. The following APIs provide an uniform way to
- * fetch these data. All arrays in bpf_prog_info are stored in a single
- * continuous memory region. This makes it easy to store the info in a
- * file.
- *
- * Before writing bpf_prog_info_linear to files, it is necessary to
- * translate pointers in bpf_prog_info to offsets. Helper functions
- * bpf_program__bpil_addr_to_offs() and bpf_program__bpil_offs_to_addr()
- * are introduced to switch between pointers and offsets.
- *
- * Examples:
- * # To fetch map_ids and prog_tags:
- * __u64 arrays = (1UL << BPF_PROG_INFO_MAP_IDS) |
- * (1UL << BPF_PROG_INFO_PROG_TAGS);
- * struct bpf_prog_info_linear *info_linear =
- * bpf_program__get_prog_info_linear(fd, arrays);
- *
- * # To save data in file
- * bpf_program__bpil_addr_to_offs(info_linear);
- * write(f, info_linear, sizeof(*info_linear) + info_linear->data_len);
- *
- * # To read data from file
- * read(f, info_linear, <proper_size>);
- * bpf_program__bpil_offs_to_addr(info_linear);
- */
-enum bpf_prog_info_array {
- BPF_PROG_INFO_FIRST_ARRAY = 0,
- BPF_PROG_INFO_JITED_INSNS = 0,
- BPF_PROG_INFO_XLATED_INSNS,
- BPF_PROG_INFO_MAP_IDS,
- BPF_PROG_INFO_JITED_KSYMS,
- BPF_PROG_INFO_JITED_FUNC_LENS,
- BPF_PROG_INFO_FUNC_INFO,
- BPF_PROG_INFO_LINE_INFO,
- BPF_PROG_INFO_JITED_LINE_INFO,
- BPF_PROG_INFO_PROG_TAGS,
- BPF_PROG_INFO_LAST_ARRAY,
-};
-
-struct bpf_prog_info_linear {
- /* size of struct bpf_prog_info, when the tool is compiled */
- __u32 info_len;
- /* total bytes allocated for data, round up to 8 bytes */
- __u32 data_len;
- /* which arrays are included in data */
- __u64 arrays;
- struct bpf_prog_info info;
- __u8 data[];
-};
-
-LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper")
-LIBBPF_API struct bpf_prog_info_linear *
-bpf_program__get_prog_info_linear(int fd, __u64 arrays);
-
-LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper")
-LIBBPF_API void
-bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear);
-
-LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper")
-LIBBPF_API void
-bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear);
-
/**
* @brief **libbpf_num_possible_cpus()** is a helper function to get the
* number of possible CPUs that the host kernel supports and expects.
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 52973cffc20c..119e6e1ea7f1 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -1,29 +1,14 @@
LIBBPF_0.0.1 {
global:
bpf_btf_get_fd_by_id;
- bpf_create_map;
- bpf_create_map_in_map;
- bpf_create_map_in_map_node;
- bpf_create_map_name;
- bpf_create_map_node;
- bpf_create_map_xattr;
- bpf_load_btf;
- bpf_load_program;
- bpf_load_program_xattr;
bpf_map__btf_key_type_id;
bpf_map__btf_value_type_id;
- bpf_map__def;
bpf_map__fd;
- bpf_map__is_offload_neutral;
bpf_map__name;
- bpf_map__next;
bpf_map__pin;
- bpf_map__prev;
- bpf_map__priv;
bpf_map__reuse_fd;
bpf_map__set_ifindex;
bpf_map__set_inner_map_fd;
- bpf_map__set_priv;
bpf_map__unpin;
bpf_map_delete_elem;
bpf_map_get_fd_by_id;
@@ -38,79 +23,37 @@ LIBBPF_0.0.1 {
bpf_object__btf_fd;
bpf_object__close;
bpf_object__find_map_by_name;
- bpf_object__find_map_by_offset;
- bpf_object__find_program_by_title;
bpf_object__kversion;
bpf_object__load;
bpf_object__name;
- bpf_object__next;
bpf_object__open;
- bpf_object__open_buffer;
- bpf_object__open_xattr;
bpf_object__pin;
bpf_object__pin_maps;
bpf_object__pin_programs;
- bpf_object__priv;
- bpf_object__set_priv;
- bpf_object__unload;
bpf_object__unpin_maps;
bpf_object__unpin_programs;
- bpf_perf_event_read_simple;
bpf_prog_attach;
bpf_prog_detach;
bpf_prog_detach2;
bpf_prog_get_fd_by_id;
bpf_prog_get_next_id;
- bpf_prog_load;
- bpf_prog_load_xattr;
bpf_prog_query;
- bpf_prog_test_run;
- bpf_prog_test_run_xattr;
bpf_program__fd;
- bpf_program__is_kprobe;
- bpf_program__is_perf_event;
- bpf_program__is_raw_tracepoint;
- bpf_program__is_sched_act;
- bpf_program__is_sched_cls;
- bpf_program__is_socket_filter;
- bpf_program__is_tracepoint;
- bpf_program__is_xdp;
- bpf_program__load;
- bpf_program__next;
- bpf_program__nth_fd;
bpf_program__pin;
- bpf_program__pin_instance;
- bpf_program__prev;
- bpf_program__priv;
bpf_program__set_expected_attach_type;
bpf_program__set_ifindex;
- bpf_program__set_kprobe;
- bpf_program__set_perf_event;
- bpf_program__set_prep;
- bpf_program__set_priv;
- bpf_program__set_raw_tracepoint;
- bpf_program__set_sched_act;
- bpf_program__set_sched_cls;
- bpf_program__set_socket_filter;
- bpf_program__set_tracepoint;
bpf_program__set_type;
- bpf_program__set_xdp;
- bpf_program__title;
bpf_program__unload;
bpf_program__unpin;
- bpf_program__unpin_instance;
bpf_prog_linfo__free;
bpf_prog_linfo__new;
bpf_prog_linfo__lfind_addr_func;
bpf_prog_linfo__lfind;
bpf_raw_tracepoint_open;
- bpf_set_link_xdp_fd;
bpf_task_fd_query;
- bpf_verify_program;
btf__fd;
btf__find_by_name;
btf__free;
- btf__get_from_id;
btf__name_by_offset;
btf__new;
btf__resolve_size;
@@ -127,48 +70,24 @@ LIBBPF_0.0.1 {
LIBBPF_0.0.2 {
global:
- bpf_probe_helper;
- bpf_probe_map_type;
- bpf_probe_prog_type;
- bpf_map__resize;
bpf_map_lookup_elem_flags;
bpf_object__btf;
bpf_object__find_map_fd_by_name;
- bpf_get_link_xdp_id;
- btf__dedup;
- btf__get_map_kv_tids;
- btf__get_nr_types;
btf__get_raw_data;
- btf__load;
btf_ext__free;
- btf_ext__func_info_rec_size;
btf_ext__get_raw_data;
- btf_ext__line_info_rec_size;
btf_ext__new;
- btf_ext__reloc_func_info;
- btf_ext__reloc_line_info;
- xsk_umem__create;
- xsk_socket__create;
- xsk_umem__delete;
- xsk_socket__delete;
- xsk_umem__fd;
- xsk_socket__fd;
- bpf_program__get_prog_info_linear;
- bpf_program__bpil_addr_to_offs;
- bpf_program__bpil_offs_to_addr;
} LIBBPF_0.0.1;
LIBBPF_0.0.3 {
global:
bpf_map__is_internal;
bpf_map_freeze;
- btf__finalize_data;
} LIBBPF_0.0.2;
LIBBPF_0.0.4 {
global:
bpf_link__destroy;
- bpf_object__load_xattr;
bpf_program__attach_kprobe;
bpf_program__attach_perf_event;
bpf_program__attach_raw_tracepoint;
@@ -176,14 +95,10 @@ LIBBPF_0.0.4 {
bpf_program__attach_uprobe;
btf_dump__dump_type;
btf_dump__free;
- btf_dump__new;
btf__parse_elf;
libbpf_num_possible_cpus;
perf_buffer__free;
- perf_buffer__new;
- perf_buffer__new_raw;
perf_buffer__poll;
- xsk_umem__create;
} LIBBPF_0.0.3;
LIBBPF_0.0.5 {
@@ -193,7 +108,6 @@ LIBBPF_0.0.5 {
LIBBPF_0.0.6 {
global:
- bpf_get_link_xdp_info;
bpf_map__get_pin_path;
bpf_map__is_pinned;
bpf_map__set_pin_path;
@@ -202,9 +116,6 @@ LIBBPF_0.0.6 {
bpf_program__attach_trace;
bpf_program__get_expected_attach_type;
bpf_program__get_type;
- bpf_program__is_tracing;
- bpf_program__set_tracing;
- bpf_program__size;
btf__find_by_name_kind;
libbpf_find_vmlinux_btf_id;
} LIBBPF_0.0.5;
@@ -224,14 +135,8 @@ LIBBPF_0.0.7 {
bpf_object__detach_skeleton;
bpf_object__load_skeleton;
bpf_object__open_skeleton;
- bpf_probe_large_insn_limit;
- bpf_prog_attach_xattr;
bpf_program__attach;
bpf_program__name;
- bpf_program__is_extension;
- bpf_program__is_struct_ops;
- bpf_program__set_extension;
- bpf_program__set_struct_ops;
btf__align_of;
libbpf_find_kernel_btf;
} LIBBPF_0.0.6;
@@ -250,10 +155,7 @@ LIBBPF_0.0.8 {
bpf_prog_attach_opts;
bpf_program__attach_cgroup;
bpf_program__attach_lsm;
- bpf_program__is_lsm;
bpf_program__set_attach_target;
- bpf_program__set_lsm;
- bpf_set_link_xdp_fd_opts;
} LIBBPF_0.0.7;
LIBBPF_0.0.9 {
@@ -291,9 +193,7 @@ LIBBPF_0.1.0 {
bpf_map__value_size;
bpf_program__attach_xdp;
bpf_program__autoload;
- bpf_program__is_sk_lookup;
bpf_program__set_autoload;
- bpf_program__set_sk_lookup;
btf__parse;
btf__parse_raw;
btf__pointer_size;
@@ -336,7 +236,6 @@ LIBBPF_0.2.0 {
perf_buffer__buffer_fd;
perf_buffer__epoll_fd;
perf_buffer__consume_buffer;
- xsk_socket__create_shared;
} LIBBPF_0.1.0;
LIBBPF_0.3.0 {
@@ -348,8 +247,6 @@ LIBBPF_0.3.0 {
btf__new_empty_split;
btf__new_split;
ring_buffer__epoll_fd;
- xsk_setup_xdp_prog;
- xsk_socket__update_xskmap;
} LIBBPF_0.2.0;
LIBBPF_0.4.0 {
@@ -397,7 +294,6 @@ LIBBPF_0.6.0 {
bpf_object__next_program;
bpf_object__prev_map;
bpf_object__prev_program;
- bpf_prog_load_deprecated;
bpf_prog_load;
bpf_program__flags;
bpf_program__insn_cnt;
@@ -407,18 +303,14 @@ LIBBPF_0.6.0 {
btf__add_decl_tag;
btf__add_type_tag;
btf__dedup;
- btf__dedup_deprecated;
btf__raw_data;
btf__type_cnt;
btf_dump__new;
- btf_dump__new_deprecated;
libbpf_major_version;
libbpf_minor_version;
libbpf_version_string;
perf_buffer__new;
- perf_buffer__new_deprecated;
perf_buffer__new_raw;
- perf_buffer__new_raw_deprecated;
} LIBBPF_0.5.0;
LIBBPF_0.7.0 {
@@ -434,10 +326,11 @@ LIBBPF_0.7.0 {
bpf_xdp_detach;
bpf_xdp_query;
bpf_xdp_query_id;
+ btf_ext__raw_data;
libbpf_probe_bpf_helper;
libbpf_probe_bpf_map_type;
libbpf_probe_bpf_prog_type;
- libbpf_set_memlock_rlim_max;
+ libbpf_set_memlock_rlim;
} LIBBPF_0.6.0;
LIBBPF_0.8.0 {
@@ -461,5 +354,15 @@ LIBBPF_0.8.0 {
} LIBBPF_0.7.0;
LIBBPF_1.0.0 {
- local: *;
+ global:
+ bpf_obj_get_opts;
+ bpf_prog_query_opts;
+ bpf_program__attach_ksyscall;
+ btf__add_enum64;
+ btf__add_enum64_value;
+ libbpf_bpf_attach_type_str;
+ libbpf_bpf_link_type_str;
+ libbpf_bpf_map_type_str;
+ libbpf_bpf_prog_type_str;
+ perf_buffer__buffer;
};
diff --git a/tools/lib/bpf/libbpf_common.h b/tools/lib/bpf/libbpf_common.h
index 000e37798ff2..9a7937f339df 100644
--- a/tools/lib/bpf/libbpf_common.h
+++ b/tools/lib/bpf/libbpf_common.h
@@ -30,20 +30,10 @@
/* Add checks for other versions below when planning deprecation of API symbols
* with the LIBBPF_DEPRECATED_SINCE macro.
*/
-#if __LIBBPF_CURRENT_VERSION_GEQ(0, 6)
-#define __LIBBPF_MARK_DEPRECATED_0_6(X) X
+#if __LIBBPF_CURRENT_VERSION_GEQ(1, 0)
+#define __LIBBPF_MARK_DEPRECATED_1_0(X) X
#else
-#define __LIBBPF_MARK_DEPRECATED_0_6(X)
-#endif
-#if __LIBBPF_CURRENT_VERSION_GEQ(0, 7)
-#define __LIBBPF_MARK_DEPRECATED_0_7(X) X
-#else
-#define __LIBBPF_MARK_DEPRECATED_0_7(X)
-#endif
-#if __LIBBPF_CURRENT_VERSION_GEQ(0, 8)
-#define __LIBBPF_MARK_DEPRECATED_0_8(X) X
-#else
-#define __LIBBPF_MARK_DEPRECATED_0_8(X)
+#define __LIBBPF_MARK_DEPRECATED_1_0(X)
#endif
/* This set of internal macros allows to do "function overloading" based on
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index 4abdbe2fea9d..4135ae0a2bc3 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -15,7 +15,6 @@
#include <linux/err.h>
#include <fcntl.h>
#include <unistd.h>
-#include "libbpf_legacy.h"
#include "relo_core.h"
/* make sure libbpf doesn't use kernel-only integer typedefs */
@@ -109,9 +108,9 @@ static inline bool str_has_sfx(const char *str, const char *sfx)
size_t str_len = strlen(str);
size_t sfx_len = strlen(sfx);
- if (sfx_len <= str_len)
- return strcmp(str + str_len - sfx_len, sfx);
- return false;
+ if (sfx_len > str_len)
+ return false;
+ return strcmp(str + str_len - sfx_len, sfx) == 0;
}
/* Symbol versioning is different between static and shared library.
@@ -351,6 +350,10 @@ enum kern_feature_id {
FEAT_MEMCG_ACCOUNT,
/* BPF cookie (bpf_get_attach_cookie() BPF helper) support */
FEAT_BPF_COOKIE,
+ /* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */
+ FEAT_BTF_ENUM64,
+ /* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */
+ FEAT_SYSCALL_WRAPPER,
__FEAT_CNT,
};
@@ -476,8 +479,6 @@ int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void
__s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
__u32 kind);
-extern enum libbpf_strict_mode libbpf_mode;
-
typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
const char *sym_name, void *ctx);
@@ -496,12 +497,8 @@ static inline int libbpf_err(int ret)
*/
static inline int libbpf_err_errno(int ret)
{
- if (libbpf_mode & LIBBPF_STRICT_DIRECT_ERRS)
- /* errno is already assumed to be set on error */
- return ret < 0 ? -errno : ret;
-
- /* legacy: on error return -1 directly and don't touch errno */
- return ret;
+ /* errno is already assumed to be set on error */
+ return ret < 0 ? -errno : ret;
}
/* handle error for pointer-returning APIs, err is assumed to be < 0 always */
@@ -509,12 +506,7 @@ static inline void *libbpf_err_ptr(int err)
{
/* set errno on error, this doesn't break anything */
errno = -err;
-
- if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS)
- return NULL;
-
- /* legacy: encode err as ptr */
- return ERR_PTR(err);
+ return NULL;
}
/* handle pointer-returning APIs' error handling */
@@ -524,11 +516,7 @@ static inline void *libbpf_ptr(void *ret)
if (IS_ERR(ret))
errno = -PTR_ERR(ret);
- if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS)
- return IS_ERR(ret) ? NULL : ret;
-
- /* legacy: pass-through original pointer */
- return ret;
+ return IS_ERR(ret) ? NULL : ret;
}
static inline bool str_is_empty(const char *s)
@@ -580,4 +568,9 @@ struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man,
const char *usdt_provider, const char *usdt_name,
__u64 usdt_cookie);
+static inline bool is_pow_of_2(size_t x)
+{
+ return x && (x & (x - 1)) == 0;
+}
+
#endif /* __LIBBPF_LIBBPF_INTERNAL_H */
diff --git a/tools/lib/bpf/libbpf_legacy.h b/tools/lib/bpf/libbpf_legacy.h
index d7bcbd01f66f..5b7e0155db6a 100644
--- a/tools/lib/bpf/libbpf_legacy.h
+++ b/tools/lib/bpf/libbpf_legacy.h
@@ -20,6 +20,11 @@
extern "C" {
#endif
+/* As of libbpf 1.0 libbpf_set_strict_mode() and enum libbpf_struct_mode have
+ * no effect. But they are left in libbpf_legacy.h so that applications that
+ * prepared for libbpf 1.0 before final release by using
+ * libbpf_set_strict_mode() still work with libbpf 1.0+ without any changes.
+ */
enum libbpf_strict_mode {
/* Turn on all supported strict features of libbpf to simulate libbpf
* v1.0 behavior.
@@ -71,8 +76,8 @@ enum libbpf_strict_mode {
* first BPF program or map creation operation. This is done only if
* kernel is too old to support memcg-based memory accounting for BPF
* subsystem. By default, RLIMIT_MEMLOCK limit is set to RLIM_INFINITY,
- * but it can be overriden with libbpf_set_memlock_rlim_max() API.
- * Note that libbpf_set_memlock_rlim_max() needs to be called before
+ * but it can be overriden with libbpf_set_memlock_rlim() API.
+ * Note that libbpf_set_memlock_rlim() needs to be called before
* the very first bpf_prog_load(), bpf_map_create() or bpf_object__load()
* operation.
*/
@@ -88,6 +93,25 @@ enum libbpf_strict_mode {
LIBBPF_API int libbpf_set_strict_mode(enum libbpf_strict_mode mode);
+/**
+ * @brief **libbpf_get_error()** extracts the error code from the passed
+ * pointer
+ * @param ptr pointer returned from libbpf API function
+ * @return error code; or 0 if no error occured
+ *
+ * Note, as of libbpf 1.0 this function is not necessary and not recommended
+ * to be used. Libbpf doesn't return error code embedded into the pointer
+ * itself. Instead, NULL is returned on error and error code is passed through
+ * thread-local errno variable. **libbpf_get_error()** is just returning -errno
+ * value if it receives NULL, which is correct only if errno hasn't been
+ * modified between libbpf API call and corresponding **libbpf_get_error()**
+ * call. Prefer to check return for NULL and use errno directly.
+ *
+ * This API is left in libbpf 1.0 to allow applications that were 1.0-ready
+ * before final libbpf 1.0 without needing to change them.
+ */
+LIBBPF_API long libbpf_get_error(const void *ptr);
+
#define DECLARE_LIBBPF_OPTS LIBBPF_OPTS
/* "Discouraged" APIs which don't follow consistent libbpf naming patterns.
diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c
index 97b06cede56f..0b5398786bf3 100644
--- a/tools/lib/bpf/libbpf_probes.c
+++ b/tools/lib/bpf/libbpf_probes.c
@@ -17,47 +17,14 @@
#include "libbpf.h"
#include "libbpf_internal.h"
-static bool grep(const char *buffer, const char *pattern)
-{
- return !!strstr(buffer, pattern);
-}
-
-static int get_vendor_id(int ifindex)
-{
- char ifname[IF_NAMESIZE], path[64], buf[8];
- ssize_t len;
- int fd;
-
- if (!if_indextoname(ifindex, ifname))
- return -1;
-
- snprintf(path, sizeof(path), "/sys/class/net/%s/device/vendor", ifname);
-
- fd = open(path, O_RDONLY | O_CLOEXEC);
- if (fd < 0)
- return -1;
-
- len = read(fd, buf, sizeof(buf));
- close(fd);
- if (len < 0)
- return -1;
- if (len >= (ssize_t)sizeof(buf))
- return -1;
- buf[len] = '\0';
-
- return strtol(buf, NULL, 0);
-}
-
static int probe_prog_load(enum bpf_prog_type prog_type,
const struct bpf_insn *insns, size_t insns_cnt,
- char *log_buf, size_t log_buf_sz,
- __u32 ifindex)
+ char *log_buf, size_t log_buf_sz)
{
LIBBPF_OPTS(bpf_prog_load_opts, opts,
.log_buf = log_buf,
.log_size = log_buf_sz,
.log_level = log_buf ? 1 : 0,
- .prog_ifindex = ifindex,
);
int fd, err, exp_err = 0;
const char *exp_msg = NULL;
@@ -161,31 +128,10 @@ int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts)
if (opts)
return libbpf_err(-EINVAL);
- ret = probe_prog_load(prog_type, insns, insn_cnt, NULL, 0, 0);
+ ret = probe_prog_load(prog_type, insns, insn_cnt, NULL, 0);
return libbpf_err(ret);
}
-bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex)
-{
- struct bpf_insn insns[2] = {
- BPF_MOV64_IMM(BPF_REG_0, 0),
- BPF_EXIT_INSN()
- };
-
- /* prefer libbpf_probe_bpf_prog_type() unless offload is requested */
- if (ifindex == 0)
- return libbpf_probe_bpf_prog_type(prog_type, NULL) == 1;
-
- if (ifindex && prog_type == BPF_PROG_TYPE_SCHED_CLS)
- /* nfp returns -EINVAL on exit(0) with TC offload */
- insns[0].imm = 2;
-
- errno = 0;
- probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), NULL, 0, ifindex);
-
- return errno != EINVAL && errno != EOPNOTSUPP;
-}
-
int libbpf__load_raw_btf(const char *raw_types, size_t types_len,
const char *str_sec, size_t str_len)
{
@@ -242,15 +188,13 @@ static int load_local_storage_btf(void)
strs, sizeof(strs));
}
-static int probe_map_create(enum bpf_map_type map_type, __u32 ifindex)
+static int probe_map_create(enum bpf_map_type map_type)
{
LIBBPF_OPTS(bpf_map_create_opts, opts);
int key_size, value_size, max_entries;
__u32 btf_key_type_id = 0, btf_value_type_id = 0;
int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err;
- opts.map_ifindex = ifindex;
-
key_size = sizeof(__u32);
value_size = sizeof(__u32);
max_entries = 1;
@@ -326,12 +270,6 @@ static int probe_map_create(enum bpf_map_type map_type, __u32 ifindex)
if (map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {
- /* TODO: probe for device, once libbpf has a function to create
- * map-in-map for offload
- */
- if (ifindex)
- goto cleanup;
-
fd_inner = bpf_map_create(BPF_MAP_TYPE_HASH, NULL,
sizeof(__u32), sizeof(__u32), 1, NULL);
if (fd_inner < 0)
@@ -370,15 +308,10 @@ int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts)
if (opts)
return libbpf_err(-EINVAL);
- ret = probe_map_create(map_type, 0);
+ ret = probe_map_create(map_type);
return libbpf_err(ret);
}
-bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
-{
- return probe_map_create(map_type, ifindex) == 1;
-}
-
int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helper_id,
const void *opts)
{
@@ -407,7 +340,7 @@ int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helpe
}
buf[0] = '\0';
- ret = probe_prog_load(prog_type, insns, insn_cnt, buf, sizeof(buf), 0);
+ ret = probe_prog_load(prog_type, insns, insn_cnt, buf, sizeof(buf));
if (ret < 0)
return libbpf_err(ret);
@@ -427,51 +360,3 @@ int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helpe
return 0;
return 1; /* assume supported */
}
-
-bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type,
- __u32 ifindex)
-{
- struct bpf_insn insns[2] = {
- BPF_EMIT_CALL(id),
- BPF_EXIT_INSN()
- };
- char buf[4096] = {};
- bool res;
-
- probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), buf, sizeof(buf), ifindex);
- res = !grep(buf, "invalid func ") && !grep(buf, "unknown func ");
-
- if (ifindex) {
- switch (get_vendor_id(ifindex)) {
- case 0x19ee: /* Netronome specific */
- res = res && !grep(buf, "not supported by FW") &&
- !grep(buf, "unsupported function id");
- break;
- default:
- break;
- }
- }
-
- return res;
-}
-
-/*
- * Probe for availability of kernel commit (5.3):
- *
- * c04c0d2b968a ("bpf: increase complexity limit and maximum program size")
- */
-bool bpf_probe_large_insn_limit(__u32 ifindex)
-{
- struct bpf_insn insns[BPF_MAXINSNS + 1];
- int i;
-
- for (i = 0; i < BPF_MAXINSNS; i++)
- insns[i] = BPF_MOV64_IMM(BPF_REG_0, 1);
- insns[BPF_MAXINSNS] = BPF_EXIT_INSN();
-
- errno = 0;
- probe_prog_load(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns), NULL, 0,
- ifindex);
-
- return errno != E2BIG && errno != EINVAL;
-}
diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c
index 9aa016fb55aa..4ac02c28e152 100644
--- a/tools/lib/bpf/linker.c
+++ b/tools/lib/bpf/linker.c
@@ -697,11 +697,6 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename,
return err;
}
-static bool is_pow_of_2(size_t x)
-{
- return x && (x & (x - 1)) == 0;
-}
-
static int linker_sanity_check_elf(struct src_obj *obj)
{
struct src_sec *sec;
@@ -1340,6 +1335,7 @@ recur:
case BTF_KIND_STRUCT:
case BTF_KIND_UNION:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
case BTF_KIND_FWD:
case BTF_KIND_FUNC:
case BTF_KIND_VAR:
@@ -1362,6 +1358,7 @@ recur:
case BTF_KIND_INT:
case BTF_KIND_FLOAT:
case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
/* ignore encoding for int and enum values for enum */
if (t1->size != t2->size) {
pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",
diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c
index cbc8967d5402..6c013168032d 100644
--- a/tools/lib/bpf/netlink.c
+++ b/tools/lib/bpf/netlink.c
@@ -27,6 +27,14 @@ typedef int (*libbpf_dump_nlmsg_t)(void *cookie, void *msg, struct nlattr **tb);
typedef int (*__dump_nlmsg_t)(struct nlmsghdr *nlmsg, libbpf_dump_nlmsg_t,
void *cookie);
+struct xdp_link_info {
+ __u32 prog_id;
+ __u32 drv_prog_id;
+ __u32 hw_prog_id;
+ __u32 skb_prog_id;
+ __u8 attach_mode;
+};
+
struct xdp_id_md {
int ifindex;
__u32 flags;
@@ -288,31 +296,6 @@ int bpf_xdp_detach(int ifindex, __u32 flags, const struct bpf_xdp_attach_opts *o
return bpf_xdp_attach(ifindex, -1, flags, opts);
}
-int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags,
- const struct bpf_xdp_set_link_opts *opts)
-{
- int old_fd = -1, ret;
-
- if (!OPTS_VALID(opts, bpf_xdp_set_link_opts))
- return libbpf_err(-EINVAL);
-
- if (OPTS_HAS(opts, old_fd)) {
- old_fd = OPTS_GET(opts, old_fd, -1);
- flags |= XDP_FLAGS_REPLACE;
- }
-
- ret = __bpf_set_link_xdp_fd_replace(ifindex, fd, old_fd, flags);
- return libbpf_err(ret);
-}
-
-int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags)
-{
- int ret;
-
- ret = __bpf_set_link_xdp_fd_replace(ifindex, fd, 0, flags);
- return libbpf_err(ret);
-}
-
static int __dump_link_nlmsg(struct nlmsghdr *nlh,
libbpf_dump_nlmsg_t dump_link_nlmsg, void *cookie)
{
@@ -413,30 +396,6 @@ int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts)
return 0;
}
-int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info,
- size_t info_size, __u32 flags)
-{
- LIBBPF_OPTS(bpf_xdp_query_opts, opts);
- size_t sz;
- int err;
-
- if (!info_size)
- return libbpf_err(-EINVAL);
-
- err = bpf_xdp_query(ifindex, flags, &opts);
- if (err)
- return libbpf_err(err);
-
- /* struct xdp_link_info field layout matches struct bpf_xdp_query_opts
- * layout after sz field
- */
- sz = min(info_size, offsetofend(struct xdp_link_info, attach_mode));
- memcpy(info, &opts.prog_id, sz);
- memset((void *)info + sz, 0, info_size - sz);
-
- return 0;
-}
-
int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id)
{
LIBBPF_OPTS(bpf_xdp_query_opts, opts);
@@ -463,11 +422,6 @@ int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id)
}
-int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags)
-{
- return bpf_xdp_query_id(ifindex, flags, prog_id);
-}
-
typedef int (*qdisc_config_t)(struct libbpf_nla_req *req);
static int clsact_config(struct libbpf_nla_req *req)
diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c
index ba4453dfd1ed..c4b0e81ae293 100644
--- a/tools/lib/bpf/relo_core.c
+++ b/tools/lib/bpf/relo_core.c
@@ -95,6 +95,7 @@ static const char *core_relo_kind_str(enum bpf_core_relo_kind kind)
case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id";
case BPF_CORE_TYPE_ID_TARGET: return "target_type_id";
case BPF_CORE_TYPE_EXISTS: return "type_exists";
+ case BPF_CORE_TYPE_MATCHES: return "type_matches";
case BPF_CORE_TYPE_SIZE: return "type_size";
case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists";
case BPF_CORE_ENUMVAL_VALUE: return "enumval_value";
@@ -123,6 +124,7 @@ static bool core_relo_is_type_based(enum bpf_core_relo_kind kind)
case BPF_CORE_TYPE_ID_LOCAL:
case BPF_CORE_TYPE_ID_TARGET:
case BPF_CORE_TYPE_EXISTS:
+ case BPF_CORE_TYPE_MATCHES:
case BPF_CORE_TYPE_SIZE:
return true;
default:
@@ -141,6 +143,86 @@ static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
}
}
+int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
+ const struct btf *targ_btf, __u32 targ_id, int level)
+{
+ const struct btf_type *local_type, *targ_type;
+ int depth = 32; /* max recursion depth */
+
+ /* caller made sure that names match (ignoring flavor suffix) */
+ local_type = btf_type_by_id(local_btf, local_id);
+ targ_type = btf_type_by_id(targ_btf, targ_id);
+ if (!btf_kind_core_compat(local_type, targ_type))
+ return 0;
+
+recur:
+ depth--;
+ if (depth < 0)
+ return -EINVAL;
+
+ local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id);
+ targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
+ if (!local_type || !targ_type)
+ return -EINVAL;
+
+ if (!btf_kind_core_compat(local_type, targ_type))
+ return 0;
+
+ switch (btf_kind(local_type)) {
+ case BTF_KIND_UNKN:
+ case BTF_KIND_STRUCT:
+ case BTF_KIND_UNION:
+ case BTF_KIND_ENUM:
+ case BTF_KIND_FWD:
+ case BTF_KIND_ENUM64:
+ return 1;
+ case BTF_KIND_INT:
+ /* just reject deprecated bitfield-like integers; all other
+ * integers are by default compatible between each other
+ */
+ return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0;
+ case BTF_KIND_PTR:
+ local_id = local_type->type;
+ targ_id = targ_type->type;
+ goto recur;
+ case BTF_KIND_ARRAY:
+ local_id = btf_array(local_type)->type;
+ targ_id = btf_array(targ_type)->type;
+ goto recur;
+ case BTF_KIND_FUNC_PROTO: {
+ struct btf_param *local_p = btf_params(local_type);
+ struct btf_param *targ_p = btf_params(targ_type);
+ __u16 local_vlen = btf_vlen(local_type);
+ __u16 targ_vlen = btf_vlen(targ_type);
+ int i, err;
+
+ if (local_vlen != targ_vlen)
+ return 0;
+
+ for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
+ if (level <= 0)
+ return -EINVAL;
+
+ skip_mods_and_typedefs(local_btf, local_p->type, &local_id);
+ skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id);
+ err = __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id,
+ level - 1);
+ if (err <= 0)
+ return err;
+ }
+
+ /* tail recurse for return type check */
+ skip_mods_and_typedefs(local_btf, local_type->type, &local_id);
+ skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id);
+ goto recur;
+ }
+ default:
+ pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
+ btf_kind_str(local_type), local_id, targ_id);
+ return 0;
+ }
+}
+
/*
* Turn bpf_core_relo into a low- and high-level spec representation,
* validating correctness along the way, as well as calculating resulting
@@ -167,11 +249,11 @@ static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
* just a parsed access string representation): [0, 1, 2, 3].
*
* High-level spec will capture only 3 points:
- * - intial zero-index access by pointer (&s->... is the same as &s[0]...);
+ * - initial zero-index access by pointer (&s->... is the same as &s[0]...);
* - field 'a' access (corresponds to '2' in low-level spec);
* - array element #3 access (corresponds to '3' in low-level spec).
*
- * Type-based relocations (TYPE_EXISTS/TYPE_SIZE,
+ * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE,
* TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their
* spec and raw_spec are kept empty.
*
@@ -186,7 +268,7 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
struct bpf_core_accessor *acc;
const struct btf_type *t;
const char *name, *spec_str;
- __u32 id;
+ __u32 id, name_off;
__s64 sz;
spec_str = btf__name_by_offset(btf, relo->access_str_off);
@@ -231,11 +313,13 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
spec->len++;
if (core_relo_is_enumval_based(relo->kind)) {
- if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
+ if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
return -EINVAL;
/* record enumerator name in a first accessor */
- acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
+ name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off
+ : btf_enum64(t)[access_idx].name_off;
+ acc->name = btf__name_by_offset(btf, name_off);
return 0;
}
@@ -340,7 +424,7 @@ recur:
if (btf_is_composite(local_type) && btf_is_composite(targ_type))
return 1;
- if (btf_kind(local_type) != btf_kind(targ_type))
+ if (!btf_kind_core_compat(local_type, targ_type))
return 0;
switch (btf_kind(local_type)) {
@@ -348,6 +432,7 @@ recur:
case BTF_KIND_FLOAT:
return 1;
case BTF_KIND_FWD:
+ case BTF_KIND_ENUM64:
case BTF_KIND_ENUM: {
const char *local_name, *targ_name;
size_t local_len, targ_len;
@@ -477,6 +562,7 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
const struct bpf_core_accessor *local_acc;
struct bpf_core_accessor *targ_acc;
int i, sz, matched;
+ __u32 name_off;
memset(targ_spec, 0, sizeof(*targ_spec));
targ_spec->btf = targ_btf;
@@ -484,9 +570,14 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
targ_spec->relo_kind = local_spec->relo_kind;
if (core_relo_is_type_based(local_spec->relo_kind)) {
- return bpf_core_types_are_compat(local_spec->btf,
- local_spec->root_type_id,
- targ_btf, targ_id);
+ if (local_spec->relo_kind == BPF_CORE_TYPE_MATCHES)
+ return bpf_core_types_match(local_spec->btf,
+ local_spec->root_type_id,
+ targ_btf, targ_id);
+ else
+ return bpf_core_types_are_compat(local_spec->btf,
+ local_spec->root_type_id,
+ targ_btf, targ_id);
}
local_acc = &local_spec->spec[0];
@@ -494,18 +585,22 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
if (core_relo_is_enumval_based(local_spec->relo_kind)) {
size_t local_essent_len, targ_essent_len;
- const struct btf_enum *e;
const char *targ_name;
/* has to resolve to an enum */
targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
- if (!btf_is_enum(targ_type))
+ if (!btf_is_any_enum(targ_type))
return 0;
local_essent_len = bpf_core_essential_name_len(local_acc->name);
- for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
- targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
+ for (i = 0; i < btf_vlen(targ_type); i++) {
+ if (btf_is_enum(targ_type))
+ name_off = btf_enum(targ_type)[i].name_off;
+ else
+ name_off = btf_enum64(targ_type)[i].name_off;
+
+ targ_name = btf__name_by_offset(targ_spec->btf, name_off);
targ_essent_len = bpf_core_essential_name_len(targ_name);
if (targ_essent_len != local_essent_len)
continue;
@@ -583,7 +678,7 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec,
static int bpf_core_calc_field_relo(const char *prog_name,
const struct bpf_core_relo *relo,
const struct bpf_core_spec *spec,
- __u32 *val, __u32 *field_sz, __u32 *type_id,
+ __u64 *val, __u32 *field_sz, __u32 *type_id,
bool *validate)
{
const struct bpf_core_accessor *acc;
@@ -680,8 +775,7 @@ static int bpf_core_calc_field_relo(const char *prog_name,
*val = byte_sz;
break;
case BPF_CORE_FIELD_SIGNED:
- /* enums will be assumed unsigned */
- *val = btf_is_enum(mt) ||
+ *val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) ||
(btf_int_encoding(mt) & BTF_INT_SIGNED);
if (validate)
*validate = true; /* signedness is never ambiguous */
@@ -708,7 +802,7 @@ static int bpf_core_calc_field_relo(const char *prog_name,
static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
const struct bpf_core_spec *spec,
- __u32 *val, bool *validate)
+ __u64 *val, bool *validate)
{
__s64 sz;
@@ -732,6 +826,7 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
*validate = false;
break;
case BPF_CORE_TYPE_EXISTS:
+ case BPF_CORE_TYPE_MATCHES:
*val = 1;
break;
case BPF_CORE_TYPE_SIZE:
@@ -751,10 +846,9 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
const struct bpf_core_spec *spec,
- __u32 *val)
+ __u64 *val)
{
const struct btf_type *t;
- const struct btf_enum *e;
switch (relo->kind) {
case BPF_CORE_ENUMVAL_EXISTS:
@@ -764,8 +858,10 @@ static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
if (!spec)
return -EUCLEAN; /* request instruction poisoning */
t = btf_type_by_id(spec->btf, spec->spec[0].type_id);
- e = btf_enum(t) + spec->spec[0].idx;
- *val = e->val;
+ if (btf_is_enum(t))
+ *val = btf_enum(t)[spec->spec[0].idx].val;
+ else
+ *val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx);
break;
default:
return -EOPNOTSUPP;
@@ -929,7 +1025,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
int insn_idx, const struct bpf_core_relo *relo,
int relo_idx, const struct bpf_core_relo_res *res)
{
- __u32 orig_val, new_val;
+ __u64 orig_val, new_val;
__u8 class;
class = BPF_CLASS(insn->code);
@@ -954,28 +1050,30 @@ poison:
if (BPF_SRC(insn->code) != BPF_K)
return -EINVAL;
if (res->validate && insn->imm != orig_val) {
- pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
+ pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n",
prog_name, relo_idx,
- insn_idx, insn->imm, orig_val, new_val);
+ insn_idx, insn->imm, (unsigned long long)orig_val,
+ (unsigned long long)new_val);
return -EINVAL;
}
orig_val = insn->imm;
insn->imm = new_val;
- pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
+ pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n",
prog_name, relo_idx, insn_idx,
- orig_val, new_val);
+ (unsigned long long)orig_val, (unsigned long long)new_val);
break;
case BPF_LDX:
case BPF_ST:
case BPF_STX:
if (res->validate && insn->off != orig_val) {
- pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
- prog_name, relo_idx, insn_idx, insn->off, orig_val, new_val);
+ pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n",
+ prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val,
+ (unsigned long long)new_val);
return -EINVAL;
}
if (new_val > SHRT_MAX) {
- pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
- prog_name, relo_idx, insn_idx, new_val);
+ pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n",
+ prog_name, relo_idx, insn_idx, (unsigned long long)new_val);
return -ERANGE;
}
if (res->fail_memsz_adjust) {
@@ -987,8 +1085,9 @@ poison:
orig_val = insn->off;
insn->off = new_val;
- pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
- prog_name, relo_idx, insn_idx, orig_val, new_val);
+ pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n",
+ prog_name, relo_idx, insn_idx, (unsigned long long)orig_val,
+ (unsigned long long)new_val);
if (res->new_sz != res->orig_sz) {
int insn_bytes_sz, insn_bpf_sz;
@@ -1024,20 +1123,20 @@ poison:
return -EINVAL;
}
- imm = insn[0].imm + ((__u64)insn[1].imm << 32);
+ imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32);
if (res->validate && imm != orig_val) {
- pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
+ pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n",
prog_name, relo_idx,
insn_idx, (unsigned long long)imm,
- orig_val, new_val);
+ (unsigned long long)orig_val, (unsigned long long)new_val);
return -EINVAL;
}
insn[0].imm = new_val;
- insn[1].imm = 0; /* currently only 32-bit values are supported */
- pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
+ insn[1].imm = new_val >> 32;
+ pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n",
prog_name, relo_idx, insn_idx,
- (unsigned long long)imm, new_val);
+ (unsigned long long)imm, (unsigned long long)new_val);
break;
}
default:
@@ -1057,7 +1156,6 @@ poison:
int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec)
{
const struct btf_type *t;
- const struct btf_enum *e;
const char *s;
__u32 type_id;
int i, len = 0;
@@ -1086,10 +1184,23 @@ int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *s
if (core_relo_is_enumval_based(spec->relo_kind)) {
t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
- e = btf_enum(t) + spec->raw_spec[0];
- s = btf__name_by_offset(spec->btf, e->name_off);
+ if (btf_is_enum(t)) {
+ const struct btf_enum *e;
+ const char *fmt_str;
+
+ e = btf_enum(t) + spec->raw_spec[0];
+ s = btf__name_by_offset(spec->btf, e->name_off);
+ fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u";
+ append_buf(fmt_str, s, e->val);
+ } else {
+ const struct btf_enum64 *e;
+ const char *fmt_str;
- append_buf("::%s = %u", s, e->val);
+ e = btf_enum64(t) + spec->raw_spec[0];
+ s = btf__name_by_offset(spec->btf, e->name_off);
+ fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu";
+ append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e));
+ }
return len;
}
@@ -1148,11 +1259,11 @@ int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *s
* 3. It is supported and expected that there might be multiple flavors
* matching the spec. As long as all the specs resolve to the same set of
* offsets across all candidates, there is no error. If there is any
- * ambiguity, CO-RE relocation will fail. This is necessary to accomodate
- * imprefection of BTF deduplication, which can cause slight duplication of
+ * ambiguity, CO-RE relocation will fail. This is necessary to accommodate
+ * imperfection of BTF deduplication, which can cause slight duplication of
* the same BTF type, if some directly or indirectly referenced (by
* pointer) type gets resolved to different actual types in different
- * object files. If such situation occurs, deduplicated BTF will end up
+ * object files. If such a situation occurs, deduplicated BTF will end up
* with two (or more) structurally identical types, which differ only in
* types they refer to through pointer. This should be OK in most cases and
* is not an error.
@@ -1261,10 +1372,12 @@ int bpf_core_calc_relo_insn(const char *prog_name,
* decision and value, otherwise it's dangerous to
* proceed due to ambiguity
*/
- pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
+ pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n",
prog_name, relo_idx,
- cand_res.poison ? "failure" : "success", cand_res.new_val,
- targ_res->poison ? "failure" : "success", targ_res->new_val);
+ cand_res.poison ? "failure" : "success",
+ (unsigned long long)cand_res.new_val,
+ targ_res->poison ? "failure" : "success",
+ (unsigned long long)targ_res->new_val);
return -EINVAL;
}
@@ -1305,3 +1418,273 @@ int bpf_core_calc_relo_insn(const char *prog_name,
return 0;
}
+
+static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off,
+ const struct btf *targ_btf, size_t targ_name_off)
+{
+ const char *local_n, *targ_n;
+ size_t local_len, targ_len;
+
+ local_n = btf__name_by_offset(local_btf, local_name_off);
+ targ_n = btf__name_by_offset(targ_btf, targ_name_off);
+
+ if (str_is_empty(targ_n))
+ return str_is_empty(local_n);
+
+ targ_len = bpf_core_essential_name_len(targ_n);
+ local_len = bpf_core_essential_name_len(local_n);
+
+ return targ_len == local_len && strncmp(local_n, targ_n, local_len) == 0;
+}
+
+static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t,
+ const struct btf *targ_btf, const struct btf_type *targ_t)
+{
+ __u16 local_vlen = btf_vlen(local_t);
+ __u16 targ_vlen = btf_vlen(targ_t);
+ int i, j;
+
+ if (local_t->size != targ_t->size)
+ return 0;
+
+ if (local_vlen > targ_vlen)
+ return 0;
+
+ /* iterate over the local enum's variants and make sure each has
+ * a symbolic name correspondent in the target
+ */
+ for (i = 0; i < local_vlen; i++) {
+ bool matched = false;
+ __u32 local_n_off, targ_n_off;
+
+ local_n_off = btf_is_enum(local_t) ? btf_enum(local_t)[i].name_off :
+ btf_enum64(local_t)[i].name_off;
+
+ for (j = 0; j < targ_vlen; j++) {
+ targ_n_off = btf_is_enum(targ_t) ? btf_enum(targ_t)[j].name_off :
+ btf_enum64(targ_t)[j].name_off;
+
+ if (bpf_core_names_match(local_btf, local_n_off, targ_btf, targ_n_off)) {
+ matched = true;
+ break;
+ }
+ }
+
+ if (!matched)
+ return 0;
+ }
+ return 1;
+}
+
+static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t,
+ const struct btf *targ_btf, const struct btf_type *targ_t,
+ bool behind_ptr, int level)
+{
+ const struct btf_member *local_m = btf_members(local_t);
+ __u16 local_vlen = btf_vlen(local_t);
+ __u16 targ_vlen = btf_vlen(targ_t);
+ int i, j, err;
+
+ if (local_vlen > targ_vlen)
+ return 0;
+
+ /* check that all local members have a match in the target */
+ for (i = 0; i < local_vlen; i++, local_m++) {
+ const struct btf_member *targ_m = btf_members(targ_t);
+ bool matched = false;
+
+ for (j = 0; j < targ_vlen; j++, targ_m++) {
+ if (!bpf_core_names_match(local_btf, local_m->name_off,
+ targ_btf, targ_m->name_off))
+ continue;
+
+ err = __bpf_core_types_match(local_btf, local_m->type, targ_btf,
+ targ_m->type, behind_ptr, level - 1);
+ if (err < 0)
+ return err;
+ if (err > 0) {
+ matched = true;
+ break;
+ }
+ }
+
+ if (!matched)
+ return 0;
+ }
+ return 1;
+}
+
+/* Check that two types "match". This function assumes that root types were
+ * already checked for name match.
+ *
+ * The matching relation is defined as follows:
+ * - modifiers and typedefs are stripped (and, hence, effectively ignored)
+ * - generally speaking types need to be of same kind (struct vs. struct, union
+ * vs. union, etc.)
+ * - exceptions are struct/union behind a pointer which could also match a
+ * forward declaration of a struct or union, respectively, and enum vs.
+ * enum64 (see below)
+ * Then, depending on type:
+ * - integers:
+ * - match if size and signedness match
+ * - arrays & pointers:
+ * - target types are recursively matched
+ * - structs & unions:
+ * - local members need to exist in target with the same name
+ * - for each member we recursively check match unless it is already behind a
+ * pointer, in which case we only check matching names and compatible kind
+ * - enums:
+ * - local variants have to have a match in target by symbolic name (but not
+ * numeric value)
+ * - size has to match (but enum may match enum64 and vice versa)
+ * - function pointers:
+ * - number and position of arguments in local type has to match target
+ * - for each argument and the return value we recursively check match
+ */
+int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf,
+ __u32 targ_id, bool behind_ptr, int level)
+{
+ const struct btf_type *local_t, *targ_t;
+ int depth = 32; /* max recursion depth */
+ __u16 local_k, targ_k;
+
+ if (level <= 0)
+ return -EINVAL;
+
+ local_t = btf_type_by_id(local_btf, local_id);
+ targ_t = btf_type_by_id(targ_btf, targ_id);
+
+recur:
+ depth--;
+ if (depth < 0)
+ return -EINVAL;
+
+ local_t = skip_mods_and_typedefs(local_btf, local_id, &local_id);
+ targ_t = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id);
+ if (!local_t || !targ_t)
+ return -EINVAL;
+
+ /* While the name check happens after typedefs are skipped, root-level
+ * typedefs would still be name-matched as that's the contract with
+ * callers.
+ */
+ if (!bpf_core_names_match(local_btf, local_t->name_off, targ_btf, targ_t->name_off))
+ return 0;
+
+ local_k = btf_kind(local_t);
+ targ_k = btf_kind(targ_t);
+
+ switch (local_k) {
+ case BTF_KIND_UNKN:
+ return local_k == targ_k;
+ case BTF_KIND_FWD: {
+ bool local_f = BTF_INFO_KFLAG(local_t->info);
+
+ if (behind_ptr) {
+ if (local_k == targ_k)
+ return local_f == BTF_INFO_KFLAG(targ_t->info);
+
+ /* for forward declarations kflag dictates whether the
+ * target is a struct (0) or union (1)
+ */
+ return (targ_k == BTF_KIND_STRUCT && !local_f) ||
+ (targ_k == BTF_KIND_UNION && local_f);
+ } else {
+ if (local_k != targ_k)
+ return 0;
+
+ /* match if the forward declaration is for the same kind */
+ return local_f == BTF_INFO_KFLAG(targ_t->info);
+ }
+ }
+ case BTF_KIND_ENUM:
+ case BTF_KIND_ENUM64:
+ if (!btf_is_any_enum(targ_t))
+ return 0;
+
+ return bpf_core_enums_match(local_btf, local_t, targ_btf, targ_t);
+ case BTF_KIND_STRUCT:
+ case BTF_KIND_UNION:
+ if (behind_ptr) {
+ bool targ_f = BTF_INFO_KFLAG(targ_t->info);
+
+ if (local_k == targ_k)
+ return 1;
+
+ if (targ_k != BTF_KIND_FWD)
+ return 0;
+
+ return (local_k == BTF_KIND_UNION) == targ_f;
+ } else {
+ if (local_k != targ_k)
+ return 0;
+
+ return bpf_core_composites_match(local_btf, local_t, targ_btf, targ_t,
+ behind_ptr, level);
+ }
+ case BTF_KIND_INT: {
+ __u8 local_sgn;
+ __u8 targ_sgn;
+
+ if (local_k != targ_k)
+ return 0;
+
+ local_sgn = btf_int_encoding(local_t) & BTF_INT_SIGNED;
+ targ_sgn = btf_int_encoding(targ_t) & BTF_INT_SIGNED;
+
+ return local_t->size == targ_t->size && local_sgn == targ_sgn;
+ }
+ case BTF_KIND_PTR:
+ if (local_k != targ_k)
+ return 0;
+
+ behind_ptr = true;
+
+ local_id = local_t->type;
+ targ_id = targ_t->type;
+ goto recur;
+ case BTF_KIND_ARRAY: {
+ const struct btf_array *local_array = btf_array(local_t);
+ const struct btf_array *targ_array = btf_array(targ_t);
+
+ if (local_k != targ_k)
+ return 0;
+
+ if (local_array->nelems != targ_array->nelems)
+ return 0;
+
+ local_id = local_array->type;
+ targ_id = targ_array->type;
+ goto recur;
+ }
+ case BTF_KIND_FUNC_PROTO: {
+ struct btf_param *local_p = btf_params(local_t);
+ struct btf_param *targ_p = btf_params(targ_t);
+ __u16 local_vlen = btf_vlen(local_t);
+ __u16 targ_vlen = btf_vlen(targ_t);
+ int i, err;
+
+ if (local_k != targ_k)
+ return 0;
+
+ if (local_vlen != targ_vlen)
+ return 0;
+
+ for (i = 0; i < local_vlen; i++, local_p++, targ_p++) {
+ err = __bpf_core_types_match(local_btf, local_p->type, targ_btf,
+ targ_p->type, behind_ptr, level - 1);
+ if (err <= 0)
+ return err;
+ }
+
+ /* tail recurse for return type check */
+ local_id = local_t->type;
+ targ_id = targ_t->type;
+ goto recur;
+ }
+ default:
+ pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n",
+ btf_kind_str(local_t), local_id, targ_id);
+ return 0;
+ }
+}
diff --git a/tools/lib/bpf/relo_core.h b/tools/lib/bpf/relo_core.h
index 073039d8ca4f..1c0566daf8e8 100644
--- a/tools/lib/bpf/relo_core.h
+++ b/tools/lib/bpf/relo_core.h
@@ -46,9 +46,9 @@ struct bpf_core_spec {
struct bpf_core_relo_res {
/* expected value in the instruction, unless validate == false */
- __u32 orig_val;
+ __u64 orig_val;
/* new value that needs to be patched up to */
- __u32 new_val;
+ __u64 new_val;
/* relocation unsuccessful, poison instruction, but don't fail load */
bool poison;
/* some relocations can't be validated against orig_val */
@@ -68,8 +68,14 @@ struct bpf_core_relo_res {
__u32 new_type_id;
};
+int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
+ const struct btf *targ_btf, __u32 targ_id, int level);
int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
const struct btf *targ_btf, __u32 targ_id);
+int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf,
+ __u32 targ_id, bool behind_ptr, int level);
+int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf,
+ __u32 targ_id);
size_t bpf_core_essential_name_len(const char *name);
diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h
index bd6f4505e7b1..70adf7b119b9 100644
--- a/tools/lib/bpf/skel_internal.h
+++ b/tools/lib/bpf/skel_internal.h
@@ -66,13 +66,13 @@ struct bpf_load_and_run_opts {
const char *errstr;
};
-long bpf_sys_bpf(__u32 cmd, void *attr, __u32 attr_size);
+long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size);
static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
unsigned int size)
{
#ifdef __KERNEL__
- return bpf_sys_bpf(cmd, attr, size);
+ return kern_sys_bpf(cmd, attr, size);
#else
return syscall(__NR_bpf, cmd, attr, size);
#endif
diff --git a/tools/lib/bpf/usdt.bpf.h b/tools/lib/bpf/usdt.bpf.h
index 4181fddb3687..4f2adc0bd6ca 100644
--- a/tools/lib/bpf/usdt.bpf.h
+++ b/tools/lib/bpf/usdt.bpf.h
@@ -6,7 +6,6 @@
#include <linux/errno.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
-#include <bpf/bpf_core_read.h>
/* Below types and maps are internal implementation details of libbpf's USDT
* support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
@@ -30,14 +29,6 @@
#ifndef BPF_USDT_MAX_IP_CNT
#define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
#endif
-/* We use BPF CO-RE to detect support for BPF cookie from BPF side. This is
- * the only dependency on CO-RE, so if it's undesirable, user can override
- * BPF_USDT_HAS_BPF_COOKIE to specify whether to BPF cookie is supported or not.
- */
-#ifndef BPF_USDT_HAS_BPF_COOKIE
-#define BPF_USDT_HAS_BPF_COOKIE \
- bpf_core_enum_value_exists(enum bpf_func_id___usdt, BPF_FUNC_get_attach_cookie___usdt)
-#endif
enum __bpf_usdt_arg_type {
BPF_USDT_ARG_CONST,
@@ -83,15 +74,12 @@ struct {
__type(value, __u32);
} __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
-/* don't rely on user's BPF code to have latest definition of bpf_func_id */
-enum bpf_func_id___usdt {
- BPF_FUNC_get_attach_cookie___usdt = 0xBAD, /* value doesn't matter */
-};
+extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig;
static __always_inline
int __bpf_usdt_spec_id(struct pt_regs *ctx)
{
- if (!BPF_USDT_HAS_BPF_COOKIE) {
+ if (!LINUX_HAS_BPF_COOKIE) {
long ip = PT_REGS_IP(ctx);
int *spec_id_ptr;
diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
index f1c9339cfbbc..d18e37982344 100644
--- a/tools/lib/bpf/usdt.c
+++ b/tools/lib/bpf/usdt.c
@@ -441,7 +441,7 @@ static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, siz
return 0;
}
-static int parse_lib_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt)
+static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt)
{
char path[PATH_MAX], line[PATH_MAX], mode[16];
size_t seg_start, seg_end, seg_off;
@@ -531,35 +531,40 @@ err_out:
return err;
}
-static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long addr, bool relative)
+static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long virtaddr)
{
struct elf_seg *seg;
int i;
- if (relative) {
- /* for shared libraries, address is relative offset and thus
- * should be fall within logical offset-based range of
- * [offset_start, offset_end)
- */
- for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
- if (seg->offset <= addr && addr < seg->offset + (seg->end - seg->start))
- return seg;
- }
- } else {
- /* for binaries, address is absolute and thus should be within
- * absolute address range of [seg_start, seg_end)
- */
- for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
- if (seg->start <= addr && addr < seg->end)
- return seg;
- }
+ /* for ELF binaries (both executables and shared libraries), we are
+ * given virtual address (absolute for executables, relative for
+ * libraries) which should match address range of [seg_start, seg_end)
+ */
+ for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
+ if (seg->start <= virtaddr && virtaddr < seg->end)
+ return seg;
}
+ return NULL;
+}
+static struct elf_seg *find_vma_seg(struct elf_seg *segs, size_t seg_cnt, long offset)
+{
+ struct elf_seg *seg;
+ int i;
+
+ /* for VMA segments from /proc/<pid>/maps file, provided "address" is
+ * actually a file offset, so should be fall within logical
+ * offset-based range of [offset_start, offset_end)
+ */
+ for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
+ if (seg->offset <= offset && offset < seg->offset + (seg->end - seg->start))
+ return seg;
+ }
return NULL;
}
-static int parse_usdt_note(Elf *elf, const char *path, long base_addr,
- GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off,
+static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr,
+ const char *data, size_t name_off, size_t desc_off,
struct usdt_note *usdt_note);
static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie);
@@ -568,8 +573,8 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie,
struct usdt_target **out_targets, size_t *out_target_cnt)
{
- size_t off, name_off, desc_off, seg_cnt = 0, lib_seg_cnt = 0, target_cnt = 0;
- struct elf_seg *segs = NULL, *lib_segs = NULL;
+ size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0;
+ struct elf_seg *segs = NULL, *vma_segs = NULL;
struct usdt_target *targets = NULL, *target;
long base_addr = 0;
Elf_Scn *notes_scn, *base_scn;
@@ -613,8 +618,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
struct elf_seg *seg = NULL;
void *tmp;
- err = parse_usdt_note(elf, path, base_addr, &nhdr,
- data->d_buf, name_off, desc_off, &note);
+ err = parse_usdt_note(elf, path, &nhdr, data->d_buf, name_off, desc_off, &note);
if (err)
goto err_out;
@@ -648,36 +652,33 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
*
* [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation
*/
- usdt_rel_ip = usdt_abs_ip = note.loc_addr;
- if (base_addr) {
+ usdt_abs_ip = note.loc_addr;
+ if (base_addr)
usdt_abs_ip += base_addr - note.base_addr;
- usdt_rel_ip += base_addr - note.base_addr;
- }
- if (ehdr.e_type == ET_EXEC) {
- /* When attaching uprobes (which what USDTs basically
- * are) kernel expects a relative IP to be specified,
- * so if we are attaching to an executable ELF binary
- * (i.e., not a shared library), we need to calculate
- * proper relative IP based on ELF's load address
- */
- seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip, false /* relative */);
- if (!seg) {
- err = -ESRCH;
- pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n",
- usdt_provider, usdt_name, path, usdt_abs_ip);
- goto err_out;
- }
- if (!seg->is_exec) {
- err = -ESRCH;
- pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n",
- path, seg->start, seg->end, usdt_provider, usdt_name,
- usdt_abs_ip);
- goto err_out;
- }
+ /* When attaching uprobes (which is what USDTs basically are)
+ * kernel expects file offset to be specified, not a relative
+ * virtual address, so we need to translate virtual address to
+ * file offset, for both ET_EXEC and ET_DYN binaries.
+ */
+ seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip);
+ if (!seg) {
+ err = -ESRCH;
+ pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n",
+ usdt_provider, usdt_name, path, usdt_abs_ip);
+ goto err_out;
+ }
+ if (!seg->is_exec) {
+ err = -ESRCH;
+ pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n",
+ path, seg->start, seg->end, usdt_provider, usdt_name,
+ usdt_abs_ip);
+ goto err_out;
+ }
+ /* translate from virtual address to file offset */
+ usdt_rel_ip = usdt_abs_ip - seg->start + seg->offset;
- usdt_rel_ip = usdt_abs_ip - (seg->start - seg->offset);
- } else if (!man->has_bpf_cookie) { /* ehdr.e_type == ET_DYN */
+ if (ehdr.e_type == ET_DYN && !man->has_bpf_cookie) {
/* If we don't have BPF cookie support but need to
* attach to a shared library, we'll need to know and
* record absolute addresses of attach points due to
@@ -697,9 +698,9 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
goto err_out;
}
- /* lib_segs are lazily initialized only if necessary */
- if (lib_seg_cnt == 0) {
- err = parse_lib_segs(pid, path, &lib_segs, &lib_seg_cnt);
+ /* vma_segs are lazily initialized only if necessary */
+ if (vma_seg_cnt == 0) {
+ err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt);
if (err) {
pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %d\n",
pid, path, err);
@@ -707,7 +708,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
}
}
- seg = find_elf_seg(lib_segs, lib_seg_cnt, usdt_rel_ip, true /* relative */);
+ seg = find_vma_seg(vma_segs, vma_seg_cnt, usdt_rel_ip);
if (!seg) {
err = -ESRCH;
pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n",
@@ -715,7 +716,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
goto err_out;
}
- usdt_abs_ip = seg->start + (usdt_rel_ip - seg->offset);
+ usdt_abs_ip = seg->start - seg->offset + usdt_rel_ip;
}
pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n",
@@ -723,7 +724,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args,
seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0);
- /* Adjust semaphore address to be a relative offset */
+ /* Adjust semaphore address to be a file offset */
if (note.sema_addr) {
if (!man->has_sema_refcnt) {
pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n",
@@ -732,7 +733,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
goto err_out;
}
- seg = find_elf_seg(segs, seg_cnt, note.sema_addr, false /* relative */);
+ seg = find_elf_seg(segs, seg_cnt, note.sema_addr);
if (!seg) {
err = -ESRCH;
pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n",
@@ -747,7 +748,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
goto err_out;
}
- usdt_sema_off = note.sema_addr - (seg->start - seg->offset);
+ usdt_sema_off = note.sema_addr - seg->start + seg->offset;
pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n",
usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ",
@@ -770,7 +771,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
target->rel_ip = usdt_rel_ip;
target->sema_off = usdt_sema_off;
- /* notes->args references strings from Elf itself, so they can
+ /* notes.args references strings from Elf itself, so they can
* be referenced safely until elf_end() call
*/
target->spec_str = note.args;
@@ -788,7 +789,7 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *
err_out:
free(segs);
- free(lib_segs);
+ free(vma_segs);
if (err < 0)
free(targets);
return err;
@@ -1089,8 +1090,8 @@ err_out:
/* Parse out USDT ELF note from '.note.stapsdt' section.
* Logic inspired by perf's code.
*/
-static int parse_usdt_note(Elf *elf, const char *path, long base_addr,
- GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off,
+static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr,
+ const char *data, size_t name_off, size_t desc_off,
struct usdt_note *note)
{
const char *provider, *name, *args;
diff --git a/tools/lib/bpf/xsk.c b/tools/lib/bpf/xsk.c
deleted file mode 100644
index af136f73b09d..000000000000
--- a/tools/lib/bpf/xsk.c
+++ /dev/null
@@ -1,1260 +0,0 @@
-// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-
-/*
- * AF_XDP user-space access library.
- *
- * Copyright(c) 2018 - 2019 Intel Corporation.
- *
- * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
- */
-
-#include <errno.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <arpa/inet.h>
-#include <asm/barrier.h>
-#include <linux/compiler.h>
-#include <linux/ethtool.h>
-#include <linux/filter.h>
-#include <linux/if_ether.h>
-#include <linux/if_packet.h>
-#include <linux/if_xdp.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/sockios.h>
-#include <net/if.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/socket.h>
-#include <sys/types.h>
-#include <linux/if_link.h>
-
-#include "bpf.h"
-#include "libbpf.h"
-#include "libbpf_internal.h"
-#include "xsk.h"
-
-/* entire xsk.h and xsk.c is going away in libbpf 1.0, so ignore all internal
- * uses of deprecated APIs
- */
-#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
-
-#ifndef SOL_XDP
- #define SOL_XDP 283
-#endif
-
-#ifndef AF_XDP
- #define AF_XDP 44
-#endif
-
-#ifndef PF_XDP
- #define PF_XDP AF_XDP
-#endif
-
-enum xsk_prog {
- XSK_PROG_FALLBACK,
- XSK_PROG_REDIRECT_FLAGS,
-};
-
-struct xsk_umem {
- struct xsk_ring_prod *fill_save;
- struct xsk_ring_cons *comp_save;
- char *umem_area;
- struct xsk_umem_config config;
- int fd;
- int refcount;
- struct list_head ctx_list;
- bool rx_ring_setup_done;
- bool tx_ring_setup_done;
-};
-
-struct xsk_ctx {
- struct xsk_ring_prod *fill;
- struct xsk_ring_cons *comp;
- __u32 queue_id;
- struct xsk_umem *umem;
- int refcount;
- int ifindex;
- struct list_head list;
- int prog_fd;
- int link_fd;
- int xsks_map_fd;
- char ifname[IFNAMSIZ];
- bool has_bpf_link;
-};
-
-struct xsk_socket {
- struct xsk_ring_cons *rx;
- struct xsk_ring_prod *tx;
- __u64 outstanding_tx;
- struct xsk_ctx *ctx;
- struct xsk_socket_config config;
- int fd;
-};
-
-struct xsk_nl_info {
- bool xdp_prog_attached;
- int ifindex;
- int fd;
-};
-
-/* Up until and including Linux 5.3 */
-struct xdp_ring_offset_v1 {
- __u64 producer;
- __u64 consumer;
- __u64 desc;
-};
-
-/* Up until and including Linux 5.3 */
-struct xdp_mmap_offsets_v1 {
- struct xdp_ring_offset_v1 rx;
- struct xdp_ring_offset_v1 tx;
- struct xdp_ring_offset_v1 fr;
- struct xdp_ring_offset_v1 cr;
-};
-
-int xsk_umem__fd(const struct xsk_umem *umem)
-{
- return umem ? umem->fd : -EINVAL;
-}
-
-int xsk_socket__fd(const struct xsk_socket *xsk)
-{
- return xsk ? xsk->fd : -EINVAL;
-}
-
-static bool xsk_page_aligned(void *buffer)
-{
- unsigned long addr = (unsigned long)buffer;
-
- return !(addr & (getpagesize() - 1));
-}
-
-static void xsk_set_umem_config(struct xsk_umem_config *cfg,
- const struct xsk_umem_config *usr_cfg)
-{
- if (!usr_cfg) {
- cfg->fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
- cfg->comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
- cfg->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
- cfg->frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
- cfg->flags = XSK_UMEM__DEFAULT_FLAGS;
- return;
- }
-
- cfg->fill_size = usr_cfg->fill_size;
- cfg->comp_size = usr_cfg->comp_size;
- cfg->frame_size = usr_cfg->frame_size;
- cfg->frame_headroom = usr_cfg->frame_headroom;
- cfg->flags = usr_cfg->flags;
-}
-
-static int xsk_set_xdp_socket_config(struct xsk_socket_config *cfg,
- const struct xsk_socket_config *usr_cfg)
-{
- if (!usr_cfg) {
- cfg->rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
- cfg->tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
- cfg->libbpf_flags = 0;
- cfg->xdp_flags = 0;
- cfg->bind_flags = 0;
- return 0;
- }
-
- if (usr_cfg->libbpf_flags & ~XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)
- return -EINVAL;
-
- cfg->rx_size = usr_cfg->rx_size;
- cfg->tx_size = usr_cfg->tx_size;
- cfg->libbpf_flags = usr_cfg->libbpf_flags;
- cfg->xdp_flags = usr_cfg->xdp_flags;
- cfg->bind_flags = usr_cfg->bind_flags;
-
- return 0;
-}
-
-static void xsk_mmap_offsets_v1(struct xdp_mmap_offsets *off)
-{
- struct xdp_mmap_offsets_v1 off_v1;
-
- /* getsockopt on a kernel <= 5.3 has no flags fields.
- * Copy over the offsets to the correct places in the >=5.4 format
- * and put the flags where they would have been on that kernel.
- */
- memcpy(&off_v1, off, sizeof(off_v1));
-
- off->rx.producer = off_v1.rx.producer;
- off->rx.consumer = off_v1.rx.consumer;
- off->rx.desc = off_v1.rx.desc;
- off->rx.flags = off_v1.rx.consumer + sizeof(__u32);
-
- off->tx.producer = off_v1.tx.producer;
- off->tx.consumer = off_v1.tx.consumer;
- off->tx.desc = off_v1.tx.desc;
- off->tx.flags = off_v1.tx.consumer + sizeof(__u32);
-
- off->fr.producer = off_v1.fr.producer;
- off->fr.consumer = off_v1.fr.consumer;
- off->fr.desc = off_v1.fr.desc;
- off->fr.flags = off_v1.fr.consumer + sizeof(__u32);
-
- off->cr.producer = off_v1.cr.producer;
- off->cr.consumer = off_v1.cr.consumer;
- off->cr.desc = off_v1.cr.desc;
- off->cr.flags = off_v1.cr.consumer + sizeof(__u32);
-}
-
-static int xsk_get_mmap_offsets(int fd, struct xdp_mmap_offsets *off)
-{
- socklen_t optlen;
- int err;
-
- optlen = sizeof(*off);
- err = getsockopt(fd, SOL_XDP, XDP_MMAP_OFFSETS, off, &optlen);
- if (err)
- return err;
-
- if (optlen == sizeof(*off))
- return 0;
-
- if (optlen == sizeof(struct xdp_mmap_offsets_v1)) {
- xsk_mmap_offsets_v1(off);
- return 0;
- }
-
- return -EINVAL;
-}
-
-static int xsk_create_umem_rings(struct xsk_umem *umem, int fd,
- struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp)
-{
- struct xdp_mmap_offsets off;
- void *map;
- int err;
-
- err = setsockopt(fd, SOL_XDP, XDP_UMEM_FILL_RING,
- &umem->config.fill_size,
- sizeof(umem->config.fill_size));
- if (err)
- return -errno;
-
- err = setsockopt(fd, SOL_XDP, XDP_UMEM_COMPLETION_RING,
- &umem->config.comp_size,
- sizeof(umem->config.comp_size));
- if (err)
- return -errno;
-
- err = xsk_get_mmap_offsets(fd, &off);
- if (err)
- return -errno;
-
- map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64),
- PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
- XDP_UMEM_PGOFF_FILL_RING);
- if (map == MAP_FAILED)
- return -errno;
-
- fill->mask = umem->config.fill_size - 1;
- fill->size = umem->config.fill_size;
- fill->producer = map + off.fr.producer;
- fill->consumer = map + off.fr.consumer;
- fill->flags = map + off.fr.flags;
- fill->ring = map + off.fr.desc;
- fill->cached_cons = umem->config.fill_size;
-
- map = mmap(NULL, off.cr.desc + umem->config.comp_size * sizeof(__u64),
- PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd,
- XDP_UMEM_PGOFF_COMPLETION_RING);
- if (map == MAP_FAILED) {
- err = -errno;
- goto out_mmap;
- }
-
- comp->mask = umem->config.comp_size - 1;
- comp->size = umem->config.comp_size;
- comp->producer = map + off.cr.producer;
- comp->consumer = map + off.cr.consumer;
- comp->flags = map + off.cr.flags;
- comp->ring = map + off.cr.desc;
-
- return 0;
-
-out_mmap:
- munmap(map, off.fr.desc + umem->config.fill_size * sizeof(__u64));
- return err;
-}
-
-DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
-int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
- __u64 size, struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp,
- const struct xsk_umem_config *usr_config)
-{
- struct xdp_umem_reg mr;
- struct xsk_umem *umem;
- int err;
-
- if (!umem_area || !umem_ptr || !fill || !comp)
- return -EFAULT;
- if (!size && !xsk_page_aligned(umem_area))
- return -EINVAL;
-
- umem = calloc(1, sizeof(*umem));
- if (!umem)
- return -ENOMEM;
-
- umem->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
- if (umem->fd < 0) {
- err = -errno;
- goto out_umem_alloc;
- }
-
- umem->umem_area = umem_area;
- INIT_LIST_HEAD(&umem->ctx_list);
- xsk_set_umem_config(&umem->config, usr_config);
-
- memset(&mr, 0, sizeof(mr));
- mr.addr = (uintptr_t)umem_area;
- mr.len = size;
- mr.chunk_size = umem->config.frame_size;
- mr.headroom = umem->config.frame_headroom;
- mr.flags = umem->config.flags;
-
- err = setsockopt(umem->fd, SOL_XDP, XDP_UMEM_REG, &mr, sizeof(mr));
- if (err) {
- err = -errno;
- goto out_socket;
- }
-
- err = xsk_create_umem_rings(umem, umem->fd, fill, comp);
- if (err)
- goto out_socket;
-
- umem->fill_save = fill;
- umem->comp_save = comp;
- *umem_ptr = umem;
- return 0;
-
-out_socket:
- close(umem->fd);
-out_umem_alloc:
- free(umem);
- return err;
-}
-
-struct xsk_umem_config_v1 {
- __u32 fill_size;
- __u32 comp_size;
- __u32 frame_size;
- __u32 frame_headroom;
-};
-
-COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
-int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area,
- __u64 size, struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp,
- const struct xsk_umem_config *usr_config)
-{
- struct xsk_umem_config config;
-
- memcpy(&config, usr_config, sizeof(struct xsk_umem_config_v1));
- config.flags = 0;
-
- return xsk_umem__create_v0_0_4(umem_ptr, umem_area, size, fill, comp,
- &config);
-}
-
-static enum xsk_prog get_xsk_prog(void)
-{
- enum xsk_prog detected = XSK_PROG_FALLBACK;
- __u32 size_out, retval, duration;
- char data_in = 0, data_out;
- struct bpf_insn insns[] = {
- BPF_LD_MAP_FD(BPF_REG_1, 0),
- BPF_MOV64_IMM(BPF_REG_2, 0),
- BPF_MOV64_IMM(BPF_REG_3, XDP_PASS),
- BPF_EMIT_CALL(BPF_FUNC_redirect_map),
- BPF_EXIT_INSN(),
- };
- int prog_fd, map_fd, ret, insn_cnt = ARRAY_SIZE(insns);
-
- map_fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, NULL, sizeof(int), sizeof(int), 1, NULL);
- if (map_fd < 0)
- return detected;
-
- insns[0].imm = map_fd;
-
- prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
- if (prog_fd < 0) {
- close(map_fd);
- return detected;
- }
-
- ret = bpf_prog_test_run(prog_fd, 0, &data_in, 1, &data_out, &size_out, &retval, &duration);
- if (!ret && retval == XDP_PASS)
- detected = XSK_PROG_REDIRECT_FLAGS;
- close(prog_fd);
- close(map_fd);
- return detected;
-}
-
-static int xsk_load_xdp_prog(struct xsk_socket *xsk)
-{
- static const int log_buf_size = 16 * 1024;
- struct xsk_ctx *ctx = xsk->ctx;
- char log_buf[log_buf_size];
- int prog_fd;
-
- /* This is the fallback C-program:
- * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
- * {
- * int ret, index = ctx->rx_queue_index;
- *
- * // A set entry here means that the correspnding queue_id
- * // has an active AF_XDP socket bound to it.
- * ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
- * if (ret > 0)
- * return ret;
- *
- * // Fallback for pre-5.3 kernels, not supporting default
- * // action in the flags parameter.
- * if (bpf_map_lookup_elem(&xsks_map, &index))
- * return bpf_redirect_map(&xsks_map, index, 0);
- * return XDP_PASS;
- * }
- */
- struct bpf_insn prog[] = {
- /* r2 = *(u32 *)(r1 + 16) */
- BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
- /* *(u32 *)(r10 - 4) = r2 */
- BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
- /* r1 = xskmap[] */
- BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
- /* r3 = XDP_PASS */
- BPF_MOV64_IMM(BPF_REG_3, 2),
- /* call bpf_redirect_map */
- BPF_EMIT_CALL(BPF_FUNC_redirect_map),
- /* if w0 != 0 goto pc+13 */
- BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
- /* r2 = r10 */
- BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
- /* r2 += -4 */
- BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
- /* r1 = xskmap[] */
- BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
- /* call bpf_map_lookup_elem */
- BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
- /* r1 = r0 */
- BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
- /* r0 = XDP_PASS */
- BPF_MOV64_IMM(BPF_REG_0, 2),
- /* if r1 == 0 goto pc+5 */
- BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
- /* r2 = *(u32 *)(r10 - 4) */
- BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
- /* r1 = xskmap[] */
- BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
- /* r3 = 0 */
- BPF_MOV64_IMM(BPF_REG_3, 0),
- /* call bpf_redirect_map */
- BPF_EMIT_CALL(BPF_FUNC_redirect_map),
- /* The jumps are to this instruction */
- BPF_EXIT_INSN(),
- };
-
- /* This is the post-5.3 kernel C-program:
- * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
- * {
- * return bpf_redirect_map(&xsks_map, ctx->rx_queue_index, XDP_PASS);
- * }
- */
- struct bpf_insn prog_redirect_flags[] = {
- /* r2 = *(u32 *)(r1 + 16) */
- BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
- /* r1 = xskmap[] */
- BPF_LD_MAP_FD(BPF_REG_1, ctx->xsks_map_fd),
- /* r3 = XDP_PASS */
- BPF_MOV64_IMM(BPF_REG_3, 2),
- /* call bpf_redirect_map */
- BPF_EMIT_CALL(BPF_FUNC_redirect_map),
- BPF_EXIT_INSN(),
- };
- size_t insns_cnt[] = {ARRAY_SIZE(prog),
- ARRAY_SIZE(prog_redirect_flags),
- };
- struct bpf_insn *progs[] = {prog, prog_redirect_flags};
- enum xsk_prog option = get_xsk_prog();
- LIBBPF_OPTS(bpf_prog_load_opts, opts,
- .log_buf = log_buf,
- .log_size = log_buf_size,
- );
-
- prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "LGPL-2.1 or BSD-2-Clause",
- progs[option], insns_cnt[option], &opts);
- if (prog_fd < 0) {
- pr_warn("BPF log buffer:\n%s", log_buf);
- return prog_fd;
- }
-
- ctx->prog_fd = prog_fd;
- return 0;
-}
-
-static int xsk_create_bpf_link(struct xsk_socket *xsk)
-{
- DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts);
- struct xsk_ctx *ctx = xsk->ctx;
- __u32 prog_id = 0;
- int link_fd;
- int err;
-
- err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
- if (err) {
- pr_warn("getting XDP prog id failed\n");
- return err;
- }
-
- /* if there's a netlink-based XDP prog loaded on interface, bail out
- * and ask user to do the removal by himself
- */
- if (prog_id) {
- pr_warn("Netlink-based XDP prog detected, please unload it in order to launch AF_XDP prog\n");
- return -EINVAL;
- }
-
- opts.flags = xsk->config.xdp_flags & ~(XDP_FLAGS_UPDATE_IF_NOEXIST | XDP_FLAGS_REPLACE);
-
- link_fd = bpf_link_create(ctx->prog_fd, ctx->ifindex, BPF_XDP, &opts);
- if (link_fd < 0) {
- pr_warn("bpf_link_create failed: %s\n", strerror(errno));
- return link_fd;
- }
-
- ctx->link_fd = link_fd;
- return 0;
-}
-
-static int xsk_get_max_queues(struct xsk_socket *xsk)
-{
- struct ethtool_channels channels = { .cmd = ETHTOOL_GCHANNELS };
- struct xsk_ctx *ctx = xsk->ctx;
- struct ifreq ifr = {};
- int fd, err, ret;
-
- fd = socket(AF_LOCAL, SOCK_DGRAM | SOCK_CLOEXEC, 0);
- if (fd < 0)
- return -errno;
-
- ifr.ifr_data = (void *)&channels;
- libbpf_strlcpy(ifr.ifr_name, ctx->ifname, IFNAMSIZ);
- err = ioctl(fd, SIOCETHTOOL, &ifr);
- if (err && errno != EOPNOTSUPP) {
- ret = -errno;
- goto out;
- }
-
- if (err) {
- /* If the device says it has no channels, then all traffic
- * is sent to a single stream, so max queues = 1.
- */
- ret = 1;
- } else {
- /* Take the max of rx, tx, combined. Drivers return
- * the number of channels in different ways.
- */
- ret = max(channels.max_rx, channels.max_tx);
- ret = max(ret, (int)channels.max_combined);
- }
-
-out:
- close(fd);
- return ret;
-}
-
-static int xsk_create_bpf_maps(struct xsk_socket *xsk)
-{
- struct xsk_ctx *ctx = xsk->ctx;
- int max_queues;
- int fd;
-
- max_queues = xsk_get_max_queues(xsk);
- if (max_queues < 0)
- return max_queues;
-
- fd = bpf_map_create(BPF_MAP_TYPE_XSKMAP, "xsks_map",
- sizeof(int), sizeof(int), max_queues, NULL);
- if (fd < 0)
- return fd;
-
- ctx->xsks_map_fd = fd;
-
- return 0;
-}
-
-static void xsk_delete_bpf_maps(struct xsk_socket *xsk)
-{
- struct xsk_ctx *ctx = xsk->ctx;
-
- bpf_map_delete_elem(ctx->xsks_map_fd, &ctx->queue_id);
- close(ctx->xsks_map_fd);
-}
-
-static int xsk_lookup_bpf_maps(struct xsk_socket *xsk)
-{
- __u32 i, *map_ids, num_maps, prog_len = sizeof(struct bpf_prog_info);
- __u32 map_len = sizeof(struct bpf_map_info);
- struct bpf_prog_info prog_info = {};
- struct xsk_ctx *ctx = xsk->ctx;
- struct bpf_map_info map_info;
- int fd, err;
-
- err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
- if (err)
- return err;
-
- num_maps = prog_info.nr_map_ids;
-
- map_ids = calloc(prog_info.nr_map_ids, sizeof(*map_ids));
- if (!map_ids)
- return -ENOMEM;
-
- memset(&prog_info, 0, prog_len);
- prog_info.nr_map_ids = num_maps;
- prog_info.map_ids = (__u64)(unsigned long)map_ids;
-
- err = bpf_obj_get_info_by_fd(ctx->prog_fd, &prog_info, &prog_len);
- if (err)
- goto out_map_ids;
-
- ctx->xsks_map_fd = -1;
-
- for (i = 0; i < prog_info.nr_map_ids; i++) {
- fd = bpf_map_get_fd_by_id(map_ids[i]);
- if (fd < 0)
- continue;
-
- memset(&map_info, 0, map_len);
- err = bpf_obj_get_info_by_fd(fd, &map_info, &map_len);
- if (err) {
- close(fd);
- continue;
- }
-
- if (!strncmp(map_info.name, "xsks_map", sizeof(map_info.name))) {
- ctx->xsks_map_fd = fd;
- break;
- }
-
- close(fd);
- }
-
- if (ctx->xsks_map_fd == -1)
- err = -ENOENT;
-
-out_map_ids:
- free(map_ids);
- return err;
-}
-
-static int xsk_set_bpf_maps(struct xsk_socket *xsk)
-{
- struct xsk_ctx *ctx = xsk->ctx;
-
- return bpf_map_update_elem(ctx->xsks_map_fd, &ctx->queue_id,
- &xsk->fd, 0);
-}
-
-static int xsk_link_lookup(int ifindex, __u32 *prog_id, int *link_fd)
-{
- struct bpf_link_info link_info;
- __u32 link_len;
- __u32 id = 0;
- int err;
- int fd;
-
- while (true) {
- err = bpf_link_get_next_id(id, &id);
- if (err) {
- if (errno == ENOENT) {
- err = 0;
- break;
- }
- pr_warn("can't get next link: %s\n", strerror(errno));
- break;
- }
-
- fd = bpf_link_get_fd_by_id(id);
- if (fd < 0) {
- if (errno == ENOENT)
- continue;
- pr_warn("can't get link by id (%u): %s\n", id, strerror(errno));
- err = -errno;
- break;
- }
-
- link_len = sizeof(struct bpf_link_info);
- memset(&link_info, 0, link_len);
- err = bpf_obj_get_info_by_fd(fd, &link_info, &link_len);
- if (err) {
- pr_warn("can't get link info: %s\n", strerror(errno));
- close(fd);
- break;
- }
- if (link_info.type == BPF_LINK_TYPE_XDP) {
- if (link_info.xdp.ifindex == ifindex) {
- *link_fd = fd;
- if (prog_id)
- *prog_id = link_info.prog_id;
- break;
- }
- }
- close(fd);
- }
-
- return err;
-}
-
-static bool xsk_probe_bpf_link(void)
-{
- LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = XDP_FLAGS_SKB_MODE);
- struct bpf_insn insns[2] = {
- BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
- BPF_EXIT_INSN()
- };
- int prog_fd, link_fd = -1, insn_cnt = ARRAY_SIZE(insns);
- int ifindex_lo = 1;
- bool ret = false;
- int err;
-
- err = xsk_link_lookup(ifindex_lo, NULL, &link_fd);
- if (err)
- return ret;
-
- if (link_fd >= 0)
- return true;
-
- prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
- if (prog_fd < 0)
- return ret;
-
- link_fd = bpf_link_create(prog_fd, ifindex_lo, BPF_XDP, &opts);
- close(prog_fd);
-
- if (link_fd >= 0) {
- ret = true;
- close(link_fd);
- }
-
- return ret;
-}
-
-static int xsk_create_xsk_struct(int ifindex, struct xsk_socket *xsk)
-{
- char ifname[IFNAMSIZ];
- struct xsk_ctx *ctx;
- char *interface;
-
- ctx = calloc(1, sizeof(*ctx));
- if (!ctx)
- return -ENOMEM;
-
- interface = if_indextoname(ifindex, &ifname[0]);
- if (!interface) {
- free(ctx);
- return -errno;
- }
-
- ctx->ifindex = ifindex;
- libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
-
- xsk->ctx = ctx;
- xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
-
- return 0;
-}
-
-static int xsk_init_xdp_res(struct xsk_socket *xsk,
- int *xsks_map_fd)
-{
- struct xsk_ctx *ctx = xsk->ctx;
- int err;
-
- err = xsk_create_bpf_maps(xsk);
- if (err)
- return err;
-
- err = xsk_load_xdp_prog(xsk);
- if (err)
- goto err_load_xdp_prog;
-
- if (ctx->has_bpf_link)
- err = xsk_create_bpf_link(xsk);
- else
- err = bpf_set_link_xdp_fd(xsk->ctx->ifindex, ctx->prog_fd,
- xsk->config.xdp_flags);
-
- if (err)
- goto err_attach_xdp_prog;
-
- if (!xsk->rx)
- return err;
-
- err = xsk_set_bpf_maps(xsk);
- if (err)
- goto err_set_bpf_maps;
-
- return err;
-
-err_set_bpf_maps:
- if (ctx->has_bpf_link)
- close(ctx->link_fd);
- else
- bpf_set_link_xdp_fd(ctx->ifindex, -1, 0);
-err_attach_xdp_prog:
- close(ctx->prog_fd);
-err_load_xdp_prog:
- xsk_delete_bpf_maps(xsk);
- return err;
-}
-
-static int xsk_lookup_xdp_res(struct xsk_socket *xsk, int *xsks_map_fd, int prog_id)
-{
- struct xsk_ctx *ctx = xsk->ctx;
- int err;
-
- ctx->prog_fd = bpf_prog_get_fd_by_id(prog_id);
- if (ctx->prog_fd < 0) {
- err = -errno;
- goto err_prog_fd;
- }
- err = xsk_lookup_bpf_maps(xsk);
- if (err)
- goto err_lookup_maps;
-
- if (!xsk->rx)
- return err;
-
- err = xsk_set_bpf_maps(xsk);
- if (err)
- goto err_set_maps;
-
- return err;
-
-err_set_maps:
- close(ctx->xsks_map_fd);
-err_lookup_maps:
- close(ctx->prog_fd);
-err_prog_fd:
- if (ctx->has_bpf_link)
- close(ctx->link_fd);
- return err;
-}
-
-static int __xsk_setup_xdp_prog(struct xsk_socket *_xdp, int *xsks_map_fd)
-{
- struct xsk_socket *xsk = _xdp;
- struct xsk_ctx *ctx = xsk->ctx;
- __u32 prog_id = 0;
- int err;
-
- if (ctx->has_bpf_link)
- err = xsk_link_lookup(ctx->ifindex, &prog_id, &ctx->link_fd);
- else
- err = bpf_get_link_xdp_id(ctx->ifindex, &prog_id, xsk->config.xdp_flags);
-
- if (err)
- return err;
-
- err = !prog_id ? xsk_init_xdp_res(xsk, xsks_map_fd) :
- xsk_lookup_xdp_res(xsk, xsks_map_fd, prog_id);
-
- if (!err && xsks_map_fd)
- *xsks_map_fd = ctx->xsks_map_fd;
-
- return err;
-}
-
-static struct xsk_ctx *xsk_get_ctx(struct xsk_umem *umem, int ifindex,
- __u32 queue_id)
-{
- struct xsk_ctx *ctx;
-
- if (list_empty(&umem->ctx_list))
- return NULL;
-
- list_for_each_entry(ctx, &umem->ctx_list, list) {
- if (ctx->ifindex == ifindex && ctx->queue_id == queue_id) {
- ctx->refcount++;
- return ctx;
- }
- }
-
- return NULL;
-}
-
-static void xsk_put_ctx(struct xsk_ctx *ctx, bool unmap)
-{
- struct xsk_umem *umem = ctx->umem;
- struct xdp_mmap_offsets off;
- int err;
-
- if (--ctx->refcount)
- return;
-
- if (!unmap)
- goto out_free;
-
- err = xsk_get_mmap_offsets(umem->fd, &off);
- if (err)
- goto out_free;
-
- munmap(ctx->fill->ring - off.fr.desc, off.fr.desc + umem->config.fill_size *
- sizeof(__u64));
- munmap(ctx->comp->ring - off.cr.desc, off.cr.desc + umem->config.comp_size *
- sizeof(__u64));
-
-out_free:
- list_del(&ctx->list);
- free(ctx);
-}
-
-static struct xsk_ctx *xsk_create_ctx(struct xsk_socket *xsk,
- struct xsk_umem *umem, int ifindex,
- const char *ifname, __u32 queue_id,
- struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp)
-{
- struct xsk_ctx *ctx;
- int err;
-
- ctx = calloc(1, sizeof(*ctx));
- if (!ctx)
- return NULL;
-
- if (!umem->fill_save) {
- err = xsk_create_umem_rings(umem, xsk->fd, fill, comp);
- if (err) {
- free(ctx);
- return NULL;
- }
- } else if (umem->fill_save != fill || umem->comp_save != comp) {
- /* Copy over rings to new structs. */
- memcpy(fill, umem->fill_save, sizeof(*fill));
- memcpy(comp, umem->comp_save, sizeof(*comp));
- }
-
- ctx->ifindex = ifindex;
- ctx->refcount = 1;
- ctx->umem = umem;
- ctx->queue_id = queue_id;
- libbpf_strlcpy(ctx->ifname, ifname, IFNAMSIZ);
-
- ctx->fill = fill;
- ctx->comp = comp;
- list_add(&ctx->list, &umem->ctx_list);
- return ctx;
-}
-
-static void xsk_destroy_xsk_struct(struct xsk_socket *xsk)
-{
- free(xsk->ctx);
- free(xsk);
-}
-
-int xsk_socket__update_xskmap(struct xsk_socket *xsk, int fd)
-{
- xsk->ctx->xsks_map_fd = fd;
- return xsk_set_bpf_maps(xsk);
-}
-
-int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd)
-{
- struct xsk_socket *xsk;
- int res;
-
- xsk = calloc(1, sizeof(*xsk));
- if (!xsk)
- return -ENOMEM;
-
- res = xsk_create_xsk_struct(ifindex, xsk);
- if (res) {
- free(xsk);
- return -EINVAL;
- }
-
- res = __xsk_setup_xdp_prog(xsk, xsks_map_fd);
-
- xsk_destroy_xsk_struct(xsk);
-
- return res;
-}
-
-int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
- const char *ifname,
- __u32 queue_id, struct xsk_umem *umem,
- struct xsk_ring_cons *rx,
- struct xsk_ring_prod *tx,
- struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp,
- const struct xsk_socket_config *usr_config)
-{
- bool unmap, rx_setup_done = false, tx_setup_done = false;
- void *rx_map = NULL, *tx_map = NULL;
- struct sockaddr_xdp sxdp = {};
- struct xdp_mmap_offsets off;
- struct xsk_socket *xsk;
- struct xsk_ctx *ctx;
- int err, ifindex;
-
- if (!umem || !xsk_ptr || !(rx || tx))
- return -EFAULT;
-
- unmap = umem->fill_save != fill;
-
- xsk = calloc(1, sizeof(*xsk));
- if (!xsk)
- return -ENOMEM;
-
- err = xsk_set_xdp_socket_config(&xsk->config, usr_config);
- if (err)
- goto out_xsk_alloc;
-
- xsk->outstanding_tx = 0;
- ifindex = if_nametoindex(ifname);
- if (!ifindex) {
- err = -errno;
- goto out_xsk_alloc;
- }
-
- if (umem->refcount++ > 0) {
- xsk->fd = socket(AF_XDP, SOCK_RAW | SOCK_CLOEXEC, 0);
- if (xsk->fd < 0) {
- err = -errno;
- goto out_xsk_alloc;
- }
- } else {
- xsk->fd = umem->fd;
- rx_setup_done = umem->rx_ring_setup_done;
- tx_setup_done = umem->tx_ring_setup_done;
- }
-
- ctx = xsk_get_ctx(umem, ifindex, queue_id);
- if (!ctx) {
- if (!fill || !comp) {
- err = -EFAULT;
- goto out_socket;
- }
-
- ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
- fill, comp);
- if (!ctx) {
- err = -ENOMEM;
- goto out_socket;
- }
- }
- xsk->ctx = ctx;
- xsk->ctx->has_bpf_link = xsk_probe_bpf_link();
-
- if (rx && !rx_setup_done) {
- err = setsockopt(xsk->fd, SOL_XDP, XDP_RX_RING,
- &xsk->config.rx_size,
- sizeof(xsk->config.rx_size));
- if (err) {
- err = -errno;
- goto out_put_ctx;
- }
- if (xsk->fd == umem->fd)
- umem->rx_ring_setup_done = true;
- }
- if (tx && !tx_setup_done) {
- err = setsockopt(xsk->fd, SOL_XDP, XDP_TX_RING,
- &xsk->config.tx_size,
- sizeof(xsk->config.tx_size));
- if (err) {
- err = -errno;
- goto out_put_ctx;
- }
- if (xsk->fd == umem->fd)
- umem->tx_ring_setup_done = true;
- }
-
- err = xsk_get_mmap_offsets(xsk->fd, &off);
- if (err) {
- err = -errno;
- goto out_put_ctx;
- }
-
- if (rx) {
- rx_map = mmap(NULL, off.rx.desc +
- xsk->config.rx_size * sizeof(struct xdp_desc),
- PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
- xsk->fd, XDP_PGOFF_RX_RING);
- if (rx_map == MAP_FAILED) {
- err = -errno;
- goto out_put_ctx;
- }
-
- rx->mask = xsk->config.rx_size - 1;
- rx->size = xsk->config.rx_size;
- rx->producer = rx_map + off.rx.producer;
- rx->consumer = rx_map + off.rx.consumer;
- rx->flags = rx_map + off.rx.flags;
- rx->ring = rx_map + off.rx.desc;
- rx->cached_prod = *rx->producer;
- rx->cached_cons = *rx->consumer;
- }
- xsk->rx = rx;
-
- if (tx) {
- tx_map = mmap(NULL, off.tx.desc +
- xsk->config.tx_size * sizeof(struct xdp_desc),
- PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
- xsk->fd, XDP_PGOFF_TX_RING);
- if (tx_map == MAP_FAILED) {
- err = -errno;
- goto out_mmap_rx;
- }
-
- tx->mask = xsk->config.tx_size - 1;
- tx->size = xsk->config.tx_size;
- tx->producer = tx_map + off.tx.producer;
- tx->consumer = tx_map + off.tx.consumer;
- tx->flags = tx_map + off.tx.flags;
- tx->ring = tx_map + off.tx.desc;
- tx->cached_prod = *tx->producer;
- /* cached_cons is r->size bigger than the real consumer pointer
- * See xsk_prod_nb_free
- */
- tx->cached_cons = *tx->consumer + xsk->config.tx_size;
- }
- xsk->tx = tx;
-
- sxdp.sxdp_family = PF_XDP;
- sxdp.sxdp_ifindex = ctx->ifindex;
- sxdp.sxdp_queue_id = ctx->queue_id;
- if (umem->refcount > 1) {
- sxdp.sxdp_flags |= XDP_SHARED_UMEM;
- sxdp.sxdp_shared_umem_fd = umem->fd;
- } else {
- sxdp.sxdp_flags = xsk->config.bind_flags;
- }
-
- err = bind(xsk->fd, (struct sockaddr *)&sxdp, sizeof(sxdp));
- if (err) {
- err = -errno;
- goto out_mmap_tx;
- }
-
- ctx->prog_fd = -1;
-
- if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
- err = __xsk_setup_xdp_prog(xsk, NULL);
- if (err)
- goto out_mmap_tx;
- }
-
- *xsk_ptr = xsk;
- umem->fill_save = NULL;
- umem->comp_save = NULL;
- return 0;
-
-out_mmap_tx:
- if (tx)
- munmap(tx_map, off.tx.desc +
- xsk->config.tx_size * sizeof(struct xdp_desc));
-out_mmap_rx:
- if (rx)
- munmap(rx_map, off.rx.desc +
- xsk->config.rx_size * sizeof(struct xdp_desc));
-out_put_ctx:
- xsk_put_ctx(ctx, unmap);
-out_socket:
- if (--umem->refcount)
- close(xsk->fd);
-out_xsk_alloc:
- free(xsk);
- return err;
-}
-
-int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
- __u32 queue_id, struct xsk_umem *umem,
- struct xsk_ring_cons *rx, struct xsk_ring_prod *tx,
- const struct xsk_socket_config *usr_config)
-{
- if (!umem)
- return -EFAULT;
-
- return xsk_socket__create_shared(xsk_ptr, ifname, queue_id, umem,
- rx, tx, umem->fill_save,
- umem->comp_save, usr_config);
-}
-
-int xsk_umem__delete(struct xsk_umem *umem)
-{
- struct xdp_mmap_offsets off;
- int err;
-
- if (!umem)
- return 0;
-
- if (umem->refcount)
- return -EBUSY;
-
- err = xsk_get_mmap_offsets(umem->fd, &off);
- if (!err && umem->fill_save && umem->comp_save) {
- munmap(umem->fill_save->ring - off.fr.desc,
- off.fr.desc + umem->config.fill_size * sizeof(__u64));
- munmap(umem->comp_save->ring - off.cr.desc,
- off.cr.desc + umem->config.comp_size * sizeof(__u64));
- }
-
- close(umem->fd);
- free(umem);
-
- return 0;
-}
-
-void xsk_socket__delete(struct xsk_socket *xsk)
-{
- size_t desc_sz = sizeof(struct xdp_desc);
- struct xdp_mmap_offsets off;
- struct xsk_umem *umem;
- struct xsk_ctx *ctx;
- int err;
-
- if (!xsk)
- return;
-
- ctx = xsk->ctx;
- umem = ctx->umem;
- if (ctx->prog_fd != -1) {
- xsk_delete_bpf_maps(xsk);
- close(ctx->prog_fd);
- if (ctx->has_bpf_link)
- close(ctx->link_fd);
- }
-
- err = xsk_get_mmap_offsets(xsk->fd, &off);
- if (!err) {
- if (xsk->rx) {
- munmap(xsk->rx->ring - off.rx.desc,
- off.rx.desc + xsk->config.rx_size * desc_sz);
- }
- if (xsk->tx) {
- munmap(xsk->tx->ring - off.tx.desc,
- off.tx.desc + xsk->config.tx_size * desc_sz);
- }
- }
-
- xsk_put_ctx(ctx, true);
-
- umem->refcount--;
- /* Do not close an fd that also has an associated umem connected
- * to it.
- */
- if (xsk->fd != umem->fd)
- close(xsk->fd);
- free(xsk);
-}
diff --git a/tools/lib/bpf/xsk.h b/tools/lib/bpf/xsk.h
deleted file mode 100644
index 64e9c57fd792..000000000000
--- a/tools/lib/bpf/xsk.h
+++ /dev/null
@@ -1,336 +0,0 @@
-/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
-
-/*
- * AF_XDP user-space access library.
- *
- * Copyright (c) 2018 - 2019 Intel Corporation.
- * Copyright (c) 2019 Facebook
- *
- * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
- */
-
-#ifndef __LIBBPF_XSK_H
-#define __LIBBPF_XSK_H
-
-#include <stdio.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <linux/if_xdp.h>
-
-#include "libbpf.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* This whole API has been deprecated and moved to libxdp that can be found at
- * https://github.com/xdp-project/xdp-tools. The APIs are exactly the same so
- * it should just be linking with libxdp instead of libbpf for this set of
- * functionality. If not, please submit a bug report on the aforementioned page.
- */
-
-/* Load-Acquire Store-Release barriers used by the XDP socket
- * library. The following macros should *NOT* be considered part of
- * the xsk.h API, and is subject to change anytime.
- *
- * LIBRARY INTERNAL
- */
-
-#define __XSK_READ_ONCE(x) (*(volatile typeof(x) *)&x)
-#define __XSK_WRITE_ONCE(x, v) (*(volatile typeof(x) *)&x) = (v)
-
-#if defined(__i386__) || defined(__x86_64__)
-# define libbpf_smp_store_release(p, v) \
- do { \
- asm volatile("" : : : "memory"); \
- __XSK_WRITE_ONCE(*p, v); \
- } while (0)
-# define libbpf_smp_load_acquire(p) \
- ({ \
- typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \
- asm volatile("" : : : "memory"); \
- ___p1; \
- })
-#elif defined(__aarch64__)
-# define libbpf_smp_store_release(p, v) \
- asm volatile ("stlr %w1, %0" : "=Q" (*p) : "r" (v) : "memory")
-# define libbpf_smp_load_acquire(p) \
- ({ \
- typeof(*p) ___p1; \
- asm volatile ("ldar %w0, %1" \
- : "=r" (___p1) : "Q" (*p) : "memory"); \
- ___p1; \
- })
-#elif defined(__riscv)
-# define libbpf_smp_store_release(p, v) \
- do { \
- asm volatile ("fence rw,w" : : : "memory"); \
- __XSK_WRITE_ONCE(*p, v); \
- } while (0)
-# define libbpf_smp_load_acquire(p) \
- ({ \
- typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \
- asm volatile ("fence r,rw" : : : "memory"); \
- ___p1; \
- })
-#endif
-
-#ifndef libbpf_smp_store_release
-#define libbpf_smp_store_release(p, v) \
- do { \
- __sync_synchronize(); \
- __XSK_WRITE_ONCE(*p, v); \
- } while (0)
-#endif
-
-#ifndef libbpf_smp_load_acquire
-#define libbpf_smp_load_acquire(p) \
- ({ \
- typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \
- __sync_synchronize(); \
- ___p1; \
- })
-#endif
-
-/* LIBRARY INTERNAL -- END */
-
-/* Do not access these members directly. Use the functions below. */
-#define DEFINE_XSK_RING(name) \
-struct name { \
- __u32 cached_prod; \
- __u32 cached_cons; \
- __u32 mask; \
- __u32 size; \
- __u32 *producer; \
- __u32 *consumer; \
- void *ring; \
- __u32 *flags; \
-}
-
-DEFINE_XSK_RING(xsk_ring_prod);
-DEFINE_XSK_RING(xsk_ring_cons);
-
-/* For a detailed explanation on the memory barriers associated with the
- * ring, please take a look at net/xdp/xsk_queue.h.
- */
-
-struct xsk_umem;
-struct xsk_socket;
-
-static inline __u64 *xsk_ring_prod__fill_addr(struct xsk_ring_prod *fill,
- __u32 idx)
-{
- __u64 *addrs = (__u64 *)fill->ring;
-
- return &addrs[idx & fill->mask];
-}
-
-static inline const __u64 *
-xsk_ring_cons__comp_addr(const struct xsk_ring_cons *comp, __u32 idx)
-{
- const __u64 *addrs = (const __u64 *)comp->ring;
-
- return &addrs[idx & comp->mask];
-}
-
-static inline struct xdp_desc *xsk_ring_prod__tx_desc(struct xsk_ring_prod *tx,
- __u32 idx)
-{
- struct xdp_desc *descs = (struct xdp_desc *)tx->ring;
-
- return &descs[idx & tx->mask];
-}
-
-static inline const struct xdp_desc *
-xsk_ring_cons__rx_desc(const struct xsk_ring_cons *rx, __u32 idx)
-{
- const struct xdp_desc *descs = (const struct xdp_desc *)rx->ring;
-
- return &descs[idx & rx->mask];
-}
-
-static inline int xsk_ring_prod__needs_wakeup(const struct xsk_ring_prod *r)
-{
- return *r->flags & XDP_RING_NEED_WAKEUP;
-}
-
-static inline __u32 xsk_prod_nb_free(struct xsk_ring_prod *r, __u32 nb)
-{
- __u32 free_entries = r->cached_cons - r->cached_prod;
-
- if (free_entries >= nb)
- return free_entries;
-
- /* Refresh the local tail pointer.
- * cached_cons is r->size bigger than the real consumer pointer so
- * that this addition can be avoided in the more frequently
- * executed code that computs free_entries in the beginning of
- * this function. Without this optimization it whould have been
- * free_entries = r->cached_prod - r->cached_cons + r->size.
- */
- r->cached_cons = libbpf_smp_load_acquire(r->consumer);
- r->cached_cons += r->size;
-
- return r->cached_cons - r->cached_prod;
-}
-
-static inline __u32 xsk_cons_nb_avail(struct xsk_ring_cons *r, __u32 nb)
-{
- __u32 entries = r->cached_prod - r->cached_cons;
-
- if (entries == 0) {
- r->cached_prod = libbpf_smp_load_acquire(r->producer);
- entries = r->cached_prod - r->cached_cons;
- }
-
- return (entries > nb) ? nb : entries;
-}
-
-static inline __u32 xsk_ring_prod__reserve(struct xsk_ring_prod *prod, __u32 nb, __u32 *idx)
-{
- if (xsk_prod_nb_free(prod, nb) < nb)
- return 0;
-
- *idx = prod->cached_prod;
- prod->cached_prod += nb;
-
- return nb;
-}
-
-static inline void xsk_ring_prod__submit(struct xsk_ring_prod *prod, __u32 nb)
-{
- /* Make sure everything has been written to the ring before indicating
- * this to the kernel by writing the producer pointer.
- */
- libbpf_smp_store_release(prod->producer, *prod->producer + nb);
-}
-
-static inline __u32 xsk_ring_cons__peek(struct xsk_ring_cons *cons, __u32 nb, __u32 *idx)
-{
- __u32 entries = xsk_cons_nb_avail(cons, nb);
-
- if (entries > 0) {
- *idx = cons->cached_cons;
- cons->cached_cons += entries;
- }
-
- return entries;
-}
-
-static inline void xsk_ring_cons__cancel(struct xsk_ring_cons *cons, __u32 nb)
-{
- cons->cached_cons -= nb;
-}
-
-static inline void xsk_ring_cons__release(struct xsk_ring_cons *cons, __u32 nb)
-{
- /* Make sure data has been read before indicating we are done
- * with the entries by updating the consumer pointer.
- */
- libbpf_smp_store_release(cons->consumer, *cons->consumer + nb);
-
-}
-
-static inline void *xsk_umem__get_data(void *umem_area, __u64 addr)
-{
- return &((char *)umem_area)[addr];
-}
-
-static inline __u64 xsk_umem__extract_addr(__u64 addr)
-{
- return addr & XSK_UNALIGNED_BUF_ADDR_MASK;
-}
-
-static inline __u64 xsk_umem__extract_offset(__u64 addr)
-{
- return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT;
-}
-
-static inline __u64 xsk_umem__add_offset_to_addr(__u64 addr)
-{
- return xsk_umem__extract_addr(addr) + xsk_umem__extract_offset(addr);
-}
-
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_umem__fd(const struct xsk_umem *umem);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_socket__fd(const struct xsk_socket *xsk);
-
-#define XSK_RING_CONS__DEFAULT_NUM_DESCS 2048
-#define XSK_RING_PROD__DEFAULT_NUM_DESCS 2048
-#define XSK_UMEM__DEFAULT_FRAME_SHIFT 12 /* 4096 bytes */
-#define XSK_UMEM__DEFAULT_FRAME_SIZE (1 << XSK_UMEM__DEFAULT_FRAME_SHIFT)
-#define XSK_UMEM__DEFAULT_FRAME_HEADROOM 0
-#define XSK_UMEM__DEFAULT_FLAGS 0
-
-struct xsk_umem_config {
- __u32 fill_size;
- __u32 comp_size;
- __u32 frame_size;
- __u32 frame_headroom;
- __u32 flags;
-};
-
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_setup_xdp_prog(int ifindex, int *xsks_map_fd);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_socket__update_xskmap(struct xsk_socket *xsk, int xsks_map_fd);
-
-/* Flags for the libbpf_flags field. */
-#define XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD (1 << 0)
-
-struct xsk_socket_config {
- __u32 rx_size;
- __u32 tx_size;
- __u32 libbpf_flags;
- __u32 xdp_flags;
- __u16 bind_flags;
-};
-
-/* Set config to NULL to get the default configuration. */
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_umem__create(struct xsk_umem **umem,
- void *umem_area, __u64 size,
- struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp,
- const struct xsk_umem_config *config);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_umem__create_v0_0_2(struct xsk_umem **umem,
- void *umem_area, __u64 size,
- struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp,
- const struct xsk_umem_config *config);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_umem__create_v0_0_4(struct xsk_umem **umem,
- void *umem_area, __u64 size,
- struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp,
- const struct xsk_umem_config *config);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_socket__create(struct xsk_socket **xsk,
- const char *ifname, __u32 queue_id,
- struct xsk_umem *umem,
- struct xsk_ring_cons *rx,
- struct xsk_ring_prod *tx,
- const struct xsk_socket_config *config);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_socket__create_shared(struct xsk_socket **xsk_ptr,
- const char *ifname,
- __u32 queue_id, struct xsk_umem *umem,
- struct xsk_ring_cons *rx,
- struct xsk_ring_prod *tx,
- struct xsk_ring_prod *fill,
- struct xsk_ring_cons *comp,
- const struct xsk_socket_config *config);
-
-/* Returns 0 for success and -EBUSY if the umem is still in use. */
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-int xsk_umem__delete(struct xsk_umem *umem);
-LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "AF_XDP support deprecated and moved to libxdp")
-void xsk_socket__delete(struct xsk_socket *xsk);
-
-#ifdef __cplusplus
-} /* extern "C" */
-#endif
-
-#endif /* __LIBBPF_XSK_H */
diff --git a/tools/lib/perf/cpumap.c b/tools/lib/perf/cpumap.c
index 384d5e076ee4..6cd0be7c1bb4 100644
--- a/tools/lib/perf/cpumap.c
+++ b/tools/lib/perf/cpumap.c
@@ -309,7 +309,7 @@ bool perf_cpu_map__has(const struct perf_cpu_map *cpus, struct perf_cpu cpu)
return perf_cpu_map__idx(cpus, cpu) != -1;
}
-struct perf_cpu perf_cpu_map__max(struct perf_cpu_map *map)
+struct perf_cpu perf_cpu_map__max(const struct perf_cpu_map *map)
{
struct perf_cpu result = {
.cpu = -1
diff --git a/tools/lib/perf/evlist.c b/tools/lib/perf/evlist.c
index e6c98a6e3908..6b1bafe267a4 100644
--- a/tools/lib/perf/evlist.c
+++ b/tools/lib/perf/evlist.c
@@ -486,6 +486,7 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
if (ops->idx)
ops->idx(evlist, evsel, mp, idx);
+ pr_debug("idx %d: mmapping fd %d\n", idx, *output);
if (ops->mmap(map, mp, *output, evlist_cpu) < 0)
return -1;
@@ -494,6 +495,7 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
if (!idx)
perf_evlist__set_mmap_first(evlist, map, overwrite);
} else {
+ pr_debug("idx %d: set output fd %d -> %d\n", idx, fd, *output);
if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0)
return -1;
@@ -520,6 +522,48 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
}
static int
+mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
+ struct perf_mmap_param *mp)
+{
+ int nr_threads = perf_thread_map__nr(evlist->threads);
+ int nr_cpus = perf_cpu_map__nr(evlist->all_cpus);
+ int cpu, thread, idx = 0;
+ int nr_mmaps = 0;
+
+ pr_debug("%s: nr cpu values (may include -1) %d nr threads %d\n",
+ __func__, nr_cpus, nr_threads);
+
+ /* per-thread mmaps */
+ for (thread = 0; thread < nr_threads; thread++, idx++) {
+ int output = -1;
+ int output_overwrite = -1;
+
+ if (mmap_per_evsel(evlist, ops, idx, mp, 0, thread, &output,
+ &output_overwrite, &nr_mmaps))
+ goto out_unmap;
+ }
+
+ /* system-wide mmaps i.e. per-cpu */
+ for (cpu = 1; cpu < nr_cpus; cpu++, idx++) {
+ int output = -1;
+ int output_overwrite = -1;
+
+ if (mmap_per_evsel(evlist, ops, idx, mp, cpu, 0, &output,
+ &output_overwrite, &nr_mmaps))
+ goto out_unmap;
+ }
+
+ if (nr_mmaps != evlist->nr_mmaps)
+ pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps);
+
+ return 0;
+
+out_unmap:
+ perf_evlist__munmap(evlist);
+ return -1;
+}
+
+static int
mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
struct perf_mmap_param *mp)
{
@@ -528,6 +572,8 @@ mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
int nr_mmaps = 0;
int cpu, thread;
+ pr_debug("%s: nr cpu values %d nr threads %d\n", __func__, nr_cpus, nr_threads);
+
for (cpu = 0; cpu < nr_cpus; cpu++) {
int output = -1;
int output_overwrite = -1;
@@ -569,6 +615,7 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist,
struct perf_evlist_mmap_ops *ops,
struct perf_mmap_param *mp)
{
+ const struct perf_cpu_map *cpus = evlist->all_cpus;
struct perf_evsel *evsel;
if (!ops || !ops->get || !ops->mmap)
@@ -588,6 +635,9 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist,
if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0)
return -ENOMEM;
+ if (perf_cpu_map__empty(cpus))
+ return mmap_per_thread(evlist, ops, mp);
+
return mmap_per_cpu(evlist, ops, mp);
}
diff --git a/tools/lib/perf/evsel.c b/tools/lib/perf/evsel.c
index 952f3520d5c2..8ce5bbd09666 100644
--- a/tools/lib/perf/evsel.c
+++ b/tools/lib/perf/evsel.c
@@ -305,6 +305,9 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
if (read_format & PERF_FORMAT_ID)
entry += sizeof(u64);
+ if (read_format & PERF_FORMAT_LOST)
+ entry += sizeof(u64);
+
if (read_format & PERF_FORMAT_GROUP) {
nr = evsel->nr_members;
size += sizeof(u64);
@@ -314,24 +317,98 @@ int perf_evsel__read_size(struct perf_evsel *evsel)
return size;
}
+/* This only reads values for the leader */
+static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx,
+ int thread, struct perf_counts_values *count)
+{
+ size_t size = perf_evsel__read_size(evsel);
+ int *fd = FD(evsel, cpu_map_idx, thread);
+ u64 read_format = evsel->attr.read_format;
+ u64 *data;
+ int idx = 1;
+
+ if (fd == NULL || *fd < 0)
+ return -EINVAL;
+
+ data = calloc(1, size);
+ if (data == NULL)
+ return -ENOMEM;
+
+ if (readn(*fd, data, size) <= 0) {
+ free(data);
+ return -errno;
+ }
+
+ /*
+ * This reads only the leader event intentionally since we don't have
+ * perf counts values for sibling events.
+ */
+ if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+ count->ena = data[idx++];
+ if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+ count->run = data[idx++];
+
+ /* value is always available */
+ count->val = data[idx++];
+ if (read_format & PERF_FORMAT_ID)
+ count->id = data[idx++];
+ if (read_format & PERF_FORMAT_LOST)
+ count->lost = data[idx++];
+
+ free(data);
+ return 0;
+}
+
+/*
+ * The perf read format is very flexible. It needs to set the proper
+ * values according to the read format.
+ */
+static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf,
+ struct perf_counts_values *count)
+{
+ u64 read_format = evsel->attr.read_format;
+ int n = 0;
+
+ count->val = buf[n++];
+
+ if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+ count->ena = buf[n++];
+
+ if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+ count->run = buf[n++];
+
+ if (read_format & PERF_FORMAT_ID)
+ count->id = buf[n++];
+
+ if (read_format & PERF_FORMAT_LOST)
+ count->lost = buf[n++];
+}
+
int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread,
struct perf_counts_values *count)
{
size_t size = perf_evsel__read_size(evsel);
int *fd = FD(evsel, cpu_map_idx, thread);
+ u64 read_format = evsel->attr.read_format;
+ struct perf_counts_values buf;
memset(count, 0, sizeof(*count));
if (fd == NULL || *fd < 0)
return -EINVAL;
+ if (read_format & PERF_FORMAT_GROUP)
+ return perf_evsel__read_group(evsel, cpu_map_idx, thread, count);
+
if (MMAP(evsel, cpu_map_idx, thread) &&
+ !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) &&
!perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count))
return 0;
- if (readn(*fd, count->values, size) <= 0)
+ if (readn(*fd, buf.values, size) <= 0)
return -errno;
+ perf_evsel__adjust_values(evsel, buf.values, count);
return 0;
}
diff --git a/tools/lib/perf/include/internal/evsel.h b/tools/lib/perf/include/internal/evsel.h
index 2a912a1f1989..a99a75d9e78f 100644
--- a/tools/lib/perf/include/internal/evsel.h
+++ b/tools/lib/perf/include/internal/evsel.h
@@ -30,6 +30,10 @@ struct perf_sample_id {
struct perf_cpu cpu;
pid_t tid;
+ /* Guest machine pid and VCPU, valid only if machine_pid is non-zero */
+ pid_t machine_pid;
+ struct perf_cpu vcpu;
+
/* Holds total ID period value for PERF_SAMPLE_READ processing. */
u64 period;
};
diff --git a/tools/lib/perf/include/perf/cpumap.h b/tools/lib/perf/include/perf/cpumap.h
index 24de795b09bb..03aceb72a783 100644
--- a/tools/lib/perf/include/perf/cpumap.h
+++ b/tools/lib/perf/include/perf/cpumap.h
@@ -23,7 +23,7 @@ LIBPERF_API void perf_cpu_map__put(struct perf_cpu_map *map);
LIBPERF_API struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, int idx);
LIBPERF_API int perf_cpu_map__nr(const struct perf_cpu_map *cpus);
LIBPERF_API bool perf_cpu_map__empty(const struct perf_cpu_map *map);
-LIBPERF_API struct perf_cpu perf_cpu_map__max(struct perf_cpu_map *map);
+LIBPERF_API struct perf_cpu perf_cpu_map__max(const struct perf_cpu_map *map);
LIBPERF_API bool perf_cpu_map__has(const struct perf_cpu_map *map, struct perf_cpu cpu);
#define perf_cpu_map__for_each_cpu(cpu, idx, cpus) \
diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h
index e7758707cadd..93bf93a59c99 100644
--- a/tools/lib/perf/include/perf/event.h
+++ b/tools/lib/perf/include/perf/event.h
@@ -6,6 +6,7 @@
#include <linux/types.h>
#include <linux/limits.h>
#include <linux/bpf.h>
+#include <linux/compiler.h>
#include <sys/types.h> /* pid_t */
#define event_contains(obj, mem) ((obj).header.size > offsetof(typeof(obj), mem))
@@ -76,7 +77,7 @@ struct perf_record_lost_samples {
};
/*
- * PERF_FORMAT_ENABLED | PERF_FORMAT_RUNNING | PERF_FORMAT_ID
+ * PERF_FORMAT_ENABLED | PERF_FORMAT_RUNNING | PERF_FORMAT_ID | PERF_FORMAT_LOST
*/
struct perf_record_read {
struct perf_event_header header;
@@ -85,6 +86,7 @@ struct perf_record_read {
__u64 time_enabled;
__u64 time_running;
__u64 id;
+ __u64 lost;
};
struct perf_record_throttle {
@@ -153,22 +155,60 @@ enum {
PERF_CPU_MAP__MASK = 1,
};
+/*
+ * Array encoding of a perf_cpu_map where nr is the number of entries in cpu[]
+ * and each entry is a value for a CPU in the map.
+ */
struct cpu_map_entries {
__u16 nr;
__u16 cpu[];
};
-struct perf_record_record_cpu_map {
+/* Bitmap encoding of a perf_cpu_map where bitmap entries are 32-bit. */
+struct perf_record_mask_cpu_map32 {
+ /* Number of mask values. */
+ __u16 nr;
+ /* Constant 4. */
+ __u16 long_size;
+ /* Bitmap data. */
+ __u32 mask[];
+};
+
+/* Bitmap encoding of a perf_cpu_map where bitmap entries are 64-bit. */
+struct perf_record_mask_cpu_map64 {
+ /* Number of mask values. */
__u16 nr;
+ /* Constant 8. */
__u16 long_size;
- unsigned long mask[];
+ /* Legacy padding. */
+ char __pad[4];
+ /* Bitmap data. */
+ __u64 mask[];
};
-struct perf_record_cpu_map_data {
+/*
+ * 'struct perf_record_cpu_map_data' is packed as unfortunately an earlier
+ * version had unaligned data and we wish to retain file format compatibility.
+ * -irogers
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpacked"
+#pragma GCC diagnostic ignored "-Wattributes"
+
+struct __packed perf_record_cpu_map_data {
__u16 type;
- char data[];
+ union {
+ /* Used when type == PERF_CPU_MAP__CPUS. */
+ struct cpu_map_entries cpus_data;
+ /* Used when type == PERF_CPU_MAP__MASK and long_size == 4. */
+ struct perf_record_mask_cpu_map32 mask32_data;
+ /* Used when type == PERF_CPU_MAP__MASK and long_size == 8. */
+ struct perf_record_mask_cpu_map64 mask64_data;
+ };
};
+#pragma GCC diagnostic pop
+
struct perf_record_cpu_map {
struct perf_event_header header;
struct perf_record_cpu_map_data data;
@@ -237,6 +277,11 @@ struct id_index_entry {
__u64 tid;
};
+struct id_index_entry_2 {
+ __u64 machine_pid;
+ __u64 vcpu;
+};
+
struct perf_record_id_index {
struct perf_event_header header;
__u64 nr;
@@ -274,6 +319,8 @@ struct perf_record_auxtrace_error {
__u64 ip;
__u64 time;
char msg[MAX_AUXTRACE_ERROR_MSG];
+ __u32 machine_pid;
+ __u32 vcpu;
};
struct perf_record_aux {
@@ -389,6 +436,7 @@ enum perf_user_event_type { /* above any possible kernel type */
PERF_RECORD_TIME_CONV = 79,
PERF_RECORD_HEADER_FEATURE = 80,
PERF_RECORD_COMPRESSED = 81,
+ PERF_RECORD_FINISHED_INIT = 82,
PERF_RECORD_HEADER_MAX
};
diff --git a/tools/lib/perf/include/perf/evsel.h b/tools/lib/perf/include/perf/evsel.h
index 699c0ed97d34..6f92204075c2 100644
--- a/tools/lib/perf/include/perf/evsel.h
+++ b/tools/lib/perf/include/perf/evsel.h
@@ -18,8 +18,10 @@ struct perf_counts_values {
uint64_t val;
uint64_t ena;
uint64_t run;
+ uint64_t id;
+ uint64_t lost;
};
- uint64_t values[3];
+ uint64_t values[5];
};
};
diff --git a/tools/lib/perf/tests/test-evsel.c b/tools/lib/perf/tests/test-evsel.c
index 89be89afb24d..a11fc51bfb68 100644
--- a/tools/lib/perf/tests/test-evsel.c
+++ b/tools/lib/perf/tests/test-evsel.c
@@ -1,10 +1,13 @@
// SPDX-License-Identifier: GPL-2.0
#include <stdarg.h>
#include <stdio.h>
+#include <string.h>
#include <linux/perf_event.h>
+#include <linux/kernel.h>
#include <perf/cpumap.h>
#include <perf/threadmap.h>
#include <perf/evsel.h>
+#include <internal/evsel.h>
#include <internal/tests.h>
#include "tests.h"
@@ -189,6 +192,163 @@ static int test_stat_user_read(int event)
return 0;
}
+static int test_stat_read_format_single(struct perf_event_attr *attr, struct perf_thread_map *threads)
+{
+ struct perf_evsel *evsel;
+ struct perf_counts_values counts;
+ volatile int count = 0x100000;
+ int err;
+
+ evsel = perf_evsel__new(attr);
+ __T("failed to create evsel", evsel);
+
+ /* skip old kernels that don't support the format */
+ err = perf_evsel__open(evsel, NULL, threads);
+ if (err < 0)
+ return 0;
+
+ while (count--) ;
+
+ memset(&counts, -1, sizeof(counts));
+ perf_evsel__read(evsel, 0, 0, &counts);
+
+ __T("failed to read value", counts.val);
+ if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+ __T("failed to read TOTAL_TIME_ENABLED", counts.ena);
+ if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+ __T("failed to read TOTAL_TIME_RUNNING", counts.run);
+ if (attr->read_format & PERF_FORMAT_ID)
+ __T("failed to read ID", counts.id);
+ if (attr->read_format & PERF_FORMAT_LOST)
+ __T("failed to read LOST", counts.lost == 0);
+
+ perf_evsel__close(evsel);
+ perf_evsel__delete(evsel);
+ return 0;
+}
+
+static int test_stat_read_format_group(struct perf_event_attr *attr, struct perf_thread_map *threads)
+{
+ struct perf_evsel *leader, *member;
+ struct perf_counts_values counts;
+ volatile int count = 0x100000;
+ int err;
+
+ attr->read_format |= PERF_FORMAT_GROUP;
+ leader = perf_evsel__new(attr);
+ __T("failed to create leader", leader);
+
+ attr->read_format &= ~PERF_FORMAT_GROUP;
+ member = perf_evsel__new(attr);
+ __T("failed to create member", member);
+
+ member->leader = leader;
+ leader->nr_members = 2;
+
+ /* skip old kernels that don't support the format */
+ err = perf_evsel__open(leader, NULL, threads);
+ if (err < 0)
+ return 0;
+ err = perf_evsel__open(member, NULL, threads);
+ if (err < 0)
+ return 0;
+
+ while (count--) ;
+
+ memset(&counts, -1, sizeof(counts));
+ perf_evsel__read(leader, 0, 0, &counts);
+
+ __T("failed to read leader value", counts.val);
+ if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+ __T("failed to read leader TOTAL_TIME_ENABLED", counts.ena);
+ if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+ __T("failed to read leader TOTAL_TIME_RUNNING", counts.run);
+ if (attr->read_format & PERF_FORMAT_ID)
+ __T("failed to read leader ID", counts.id);
+ if (attr->read_format & PERF_FORMAT_LOST)
+ __T("failed to read leader LOST", counts.lost == 0);
+
+ memset(&counts, -1, sizeof(counts));
+ perf_evsel__read(member, 0, 0, &counts);
+
+ __T("failed to read member value", counts.val);
+ if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
+ __T("failed to read member TOTAL_TIME_ENABLED", counts.ena);
+ if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
+ __T("failed to read member TOTAL_TIME_RUNNING", counts.run);
+ if (attr->read_format & PERF_FORMAT_ID)
+ __T("failed to read member ID", counts.id);
+ if (attr->read_format & PERF_FORMAT_LOST)
+ __T("failed to read member LOST", counts.lost == 0);
+
+ perf_evsel__close(member);
+ perf_evsel__close(leader);
+ perf_evsel__delete(member);
+ perf_evsel__delete(leader);
+ return 0;
+}
+
+static int test_stat_read_format(void)
+{
+ struct perf_thread_map *threads;
+ struct perf_event_attr attr = {
+ .type = PERF_TYPE_SOFTWARE,
+ .config = PERF_COUNT_SW_TASK_CLOCK,
+ };
+ int err, i;
+
+#define FMT(_fmt) PERF_FORMAT_ ## _fmt
+#define FMT_TIME (FMT(TOTAL_TIME_ENABLED) | FMT(TOTAL_TIME_RUNNING))
+
+ uint64_t test_formats [] = {
+ 0,
+ FMT_TIME,
+ FMT(ID),
+ FMT(LOST),
+ FMT_TIME | FMT(ID),
+ FMT_TIME | FMT(LOST),
+ FMT_TIME | FMT(ID) | FMT(LOST),
+ FMT(ID) | FMT(LOST),
+ };
+
+#undef FMT
+#undef FMT_TIME
+
+ threads = perf_thread_map__new_dummy();
+ __T("failed to create threads", threads);
+
+ perf_thread_map__set_pid(threads, 0, 0);
+
+ for (i = 0; i < (int)ARRAY_SIZE(test_formats); i++) {
+ attr.read_format = test_formats[i];
+ __T_VERBOSE("testing single read with read_format: %lx\n",
+ (unsigned long)test_formats[i]);
+
+ err = test_stat_read_format_single(&attr, threads);
+ __T("failed to read single format", err == 0);
+ }
+
+ perf_thread_map__put(threads);
+
+ threads = perf_thread_map__new_array(2, NULL);
+ __T("failed to create threads", threads);
+
+ perf_thread_map__set_pid(threads, 0, 0);
+ perf_thread_map__set_pid(threads, 1, 0);
+
+ for (i = 0; i < (int)ARRAY_SIZE(test_formats); i++) {
+ attr.read_format = test_formats[i];
+ __T_VERBOSE("testing group read with read_format: %lx\n",
+ (unsigned long)test_formats[i]);
+
+ err = test_stat_read_format_group(&attr, threads);
+ __T("failed to read group format", err == 0);
+ }
+
+ perf_thread_map__put(threads);
+ return 0;
+}
+
int test_evsel(int argc, char **argv)
{
__T_START;
@@ -200,6 +360,7 @@ int test_evsel(int argc, char **argv)
test_stat_thread_enable();
test_stat_user_read(PERF_COUNT_HW_INSTRUCTIONS);
test_stat_user_read(PERF_COUNT_HW_CPU_CYCLES);
+ test_stat_read_format();
__T_END;
return tests_failed == 0 ? 0 : -1;