diff options
Diffstat (limited to 'tools')
40 files changed, 3202 insertions, 331 deletions
diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst index f55a2daed59b..9e827e342d9e 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-map.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst @@ -42,7 +42,8 @@ MAP COMMANDS | | **percpu_array** | **stack_trace** | **cgroup_array** | **lru_hash** | | **lru_percpu_hash** | **lpm_trie** | **array_of_maps** | **hash_of_maps** | | **devmap** | **sockmap** | **cpumap** | **xskmap** | **sockhash** -| | **cgroup_storage** | **reuseport_sockarray** | **percpu_cgroup_storage** } +| | **cgroup_storage** | **reuseport_sockarray** | **percpu_cgroup_storage** +| | **queue** | **stack** } DESCRIPTION =========== diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index ac4e904b10fb..8db78ed82a71 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -15,7 +15,8 @@ SYNOPSIS *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-f** | **--bpffs** } } *COMMANDS* := - { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** | **help** } + { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** + | **loadall** | **help** } MAP COMMANDS ============= @@ -24,9 +25,9 @@ MAP COMMANDS | **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes** | **visual**}] | **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes**}] | **bpftool** **prog pin** *PROG* *FILE* -| **bpftool** **prog load** *OBJ* *FILE* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*] -| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* *MAP* -| **bpftool** **prog detach** *PROG* *ATTACH_TYPE* *MAP* +| **bpftool** **prog { load | loadall }** *OBJ* *PATH* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*] +| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*] +| **bpftool** **prog detach** *PROG* *ATTACH_TYPE* [*MAP*] | **bpftool** **prog help** | | *MAP* := { **id** *MAP_ID* | **pinned** *FILE* } @@ -39,7 +40,9 @@ MAP COMMANDS | **cgroup/bind4** | **cgroup/bind6** | **cgroup/post_bind4** | **cgroup/post_bind6** | | **cgroup/connect4** | **cgroup/connect6** | **cgroup/sendmsg4** | **cgroup/sendmsg6** | } -| *ATTACH_TYPE* := { **msg_verdict** | **skb_verdict** | **skb_parse** } +| *ATTACH_TYPE* := { +| **msg_verdict** | **skb_verdict** | **skb_parse** | **flow_dissector** +| } DESCRIPTION @@ -79,8 +82,11 @@ DESCRIPTION contain a dot character ('.'), which is reserved for future extensions of *bpffs*. - **bpftool prog load** *OBJ* *FILE* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*] - Load bpf program from binary *OBJ* and pin as *FILE*. + **bpftool prog { load | loadall }** *OBJ* *PATH* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*] [**pinmaps** *MAP_DIR*] + Load bpf program(s) from binary *OBJ* and pin as *PATH*. + **bpftool prog load** pins only the first program from the + *OBJ* as *PATH*. **bpftool prog loadall** pins all programs + from the *OBJ* under *PATH* directory. **type** is optional, if not specified program type will be inferred from section names. By default bpftool will create new maps as declared in the ELF @@ -92,18 +98,24 @@ DESCRIPTION use, referring to it by **id** or through a **pinned** file. If **dev** *NAME* is specified program will be loaded onto given networking device (offload). + Optional **pinmaps** argument can be provided to pin all + maps under *MAP_DIR* directory. - Note: *FILE* must be located in *bpffs* mount. It must not + Note: *PATH* must be located in *bpffs* mount. It must not contain a dot character ('.'), which is reserved for future extensions of *bpffs*. - **bpftool prog attach** *PROG* *ATTACH_TYPE* *MAP* - Attach bpf program *PROG* (with type specified by *ATTACH_TYPE*) - to the map *MAP*. - - **bpftool prog detach** *PROG* *ATTACH_TYPE* *MAP* - Detach bpf program *PROG* (with type specified by *ATTACH_TYPE*) - from the map *MAP*. + **bpftool prog attach** *PROG* *ATTACH_TYPE* [*MAP*] + Attach bpf program *PROG* (with type specified by + *ATTACH_TYPE*). Most *ATTACH_TYPEs* require a *MAP* + parameter, with the exception of *flow_dissector* which is + attached to current networking name space. + + **bpftool prog detach** *PROG* *ATTACH_TYPE* [*MAP*] + Detach bpf program *PROG* (with type specified by + *ATTACH_TYPE*). Most *ATTACH_TYPEs* require a *MAP* + parameter, with the exception of *flow_dissector* which is + detached from the current networking name space. **bpftool prog help** Print short help message. diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index dac7eff4c7e5..1bea6b979082 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -53,7 +53,7 @@ ifneq ($(EXTRA_LDFLAGS),) LDFLAGS += $(EXTRA_LDFLAGS) endif -LIBS = -lelf -lbfd -lopcodes $(LIBBPF) +LIBS = -lelf $(LIBBPF) INSTALL ?= install RM ?= rm -f @@ -90,7 +90,16 @@ include $(wildcard $(OUTPUT)*.d) all: $(OUTPUT)bpftool -SRCS = $(wildcard *.c) +BFD_SRCS = jit_disasm.c + +SRCS = $(filter-out $(BFD_SRCS),$(wildcard *.c)) + +ifeq ($(feature-libbfd),1) +CFLAGS += -DHAVE_LIBBFD_SUPPORT +SRCS += $(BFD_SRCS) +LIBS += -lbfd -lopcodes +endif + OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o $(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 3f78e6404589..45c2db257d2b 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -243,7 +243,7 @@ _bpftool() # Completion depends on object and command in use case $object in prog) - if [[ $command != "load" ]]; then + if [[ $command != "load" && $command != "loadall" ]]; then case $prev in id) _bpftool_get_prog_ids @@ -299,7 +299,8 @@ _bpftool() fi if [[ ${#words[@]} == 6 ]]; then - COMPREPLY=( $( compgen -W "msg_verdict skb_verdict skb_parse" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "msg_verdict skb_verdict \ + skb_parse flow_dissector" -- "$cur" ) ) return 0 fi @@ -309,7 +310,7 @@ _bpftool() fi return 0 ;; - load) + load|loadall) local obj if [[ ${#words[@]} -lt 6 ]]; then @@ -338,7 +339,16 @@ _bpftool() case $prev in type) - COMPREPLY=( $( compgen -W "socket kprobe kretprobe classifier action tracepoint raw_tracepoint xdp perf_event cgroup/skb cgroup/sock cgroup/dev lwt_in lwt_out lwt_xmit lwt_seg6local sockops sk_skb sk_msg lirc_mode2 cgroup/bind4 cgroup/bind6 cgroup/connect4 cgroup/connect6 cgroup/sendmsg4 cgroup/sendmsg6 cgroup/post_bind4 cgroup/post_bind6" -- \ + COMPREPLY=( $( compgen -W "socket kprobe \ + kretprobe classifier flow_dissector \ + action tracepoint raw_tracepoint \ + xdp perf_event cgroup/skb cgroup/sock \ + cgroup/dev lwt_in lwt_out lwt_xmit \ + lwt_seg6local sockops sk_skb sk_msg \ + lirc_mode2 cgroup/bind4 cgroup/bind6 \ + cgroup/connect4 cgroup/connect6 \ + cgroup/sendmsg4 cgroup/sendmsg6 \ + cgroup/post_bind4 cgroup/post_bind6" -- \ "$cur" ) ) return 0 ;; @@ -346,7 +356,7 @@ _bpftool() _bpftool_get_map_ids return 0 ;; - pinned) + pinned|pinmaps) _filedir return 0 ;; @@ -358,6 +368,7 @@ _bpftool() COMPREPLY=( $( compgen -W "map" -- "$cur" ) ) _bpftool_once_attr 'type' _bpftool_once_attr 'dev' + _bpftool_once_attr 'pinmaps' return 0 ;; esac diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 55bc512a1831..c3fd3a7cb787 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -249,3 +249,139 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id, { return btf_dumper_do_type(d, type_id, 0, data); } + +#define BTF_PRINT_ARG(...) \ + do { \ + pos += snprintf(func_sig + pos, size - pos, \ + __VA_ARGS__); \ + if (pos >= size) \ + return -1; \ + } while (0) +#define BTF_PRINT_TYPE(type) \ + do { \ + pos = __btf_dumper_type_only(btf, type, func_sig, \ + pos, size); \ + if (pos == -1) \ + return -1; \ + } while (0) + +static int btf_dump_func(const struct btf *btf, char *func_sig, + const struct btf_type *func_proto, + const struct btf_type *func, int pos, int size); + +static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, + char *func_sig, int pos, int size) +{ + const struct btf_type *proto_type; + const struct btf_array *array; + const struct btf_type *t; + + if (!type_id) { + BTF_PRINT_ARG("void "); + return pos; + } + + t = btf__type_by_id(btf, type_id); + + switch (BTF_INFO_KIND(t->info)) { + case BTF_KIND_INT: + BTF_PRINT_ARG("%s ", btf__name_by_offset(btf, t->name_off)); + break; + case BTF_KIND_STRUCT: + BTF_PRINT_ARG("struct %s ", + btf__name_by_offset(btf, t->name_off)); + break; + case BTF_KIND_UNION: + BTF_PRINT_ARG("union %s ", + btf__name_by_offset(btf, t->name_off)); + break; + case BTF_KIND_ENUM: + BTF_PRINT_ARG("enum %s ", + btf__name_by_offset(btf, t->name_off)); + break; + case BTF_KIND_ARRAY: + array = (struct btf_array *)(t + 1); + BTF_PRINT_TYPE(array->type); + BTF_PRINT_ARG("[%d]", array->nelems); + break; + case BTF_KIND_PTR: + BTF_PRINT_TYPE(t->type); + BTF_PRINT_ARG("* "); + break; + case BTF_KIND_UNKN: + case BTF_KIND_FWD: + case BTF_KIND_TYPEDEF: + return -1; + case BTF_KIND_VOLATILE: + BTF_PRINT_ARG("volatile "); + BTF_PRINT_TYPE(t->type); + break; + case BTF_KIND_CONST: + BTF_PRINT_ARG("const "); + BTF_PRINT_TYPE(t->type); + break; + case BTF_KIND_RESTRICT: + BTF_PRINT_ARG("restrict "); + BTF_PRINT_TYPE(t->type); + break; + case BTF_KIND_FUNC_PROTO: + pos = btf_dump_func(btf, func_sig, t, NULL, pos, size); + if (pos == -1) + return -1; + break; + case BTF_KIND_FUNC: + proto_type = btf__type_by_id(btf, t->type); + pos = btf_dump_func(btf, func_sig, proto_type, t, pos, size); + if (pos == -1) + return -1; + break; + default: + return -1; + } + + return pos; +} + +static int btf_dump_func(const struct btf *btf, char *func_sig, + const struct btf_type *func_proto, + const struct btf_type *func, int pos, int size) +{ + int i, vlen; + + BTF_PRINT_TYPE(func_proto->type); + if (func) + BTF_PRINT_ARG("%s(", btf__name_by_offset(btf, func->name_off)); + else + BTF_PRINT_ARG("("); + vlen = BTF_INFO_VLEN(func_proto->info); + for (i = 0; i < vlen; i++) { + struct btf_param *arg = &((struct btf_param *)(func_proto + 1))[i]; + + if (i) + BTF_PRINT_ARG(", "); + if (arg->type) { + BTF_PRINT_TYPE(arg->type); + BTF_PRINT_ARG("%s", + btf__name_by_offset(btf, arg->name_off)); + } else { + BTF_PRINT_ARG("..."); + } + } + BTF_PRINT_ARG(")"); + + return pos; +} + +void btf_dumper_type_only(const struct btf *btf, __u32 type_id, char *func_sig, + int size) +{ + int err; + + func_sig[0] = '\0'; + if (!btf) + return; + + err = __btf_dumper_type_only(btf, type_id, func_sig, 0, size); + if (err < 0) + func_sig[0] = '\0'; +} diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 25af85304ebe..cb06a5b6e016 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -46,6 +46,7 @@ #include <linux/magic.h> #include <net/if.h> #include <sys/mount.h> +#include <sys/resource.h> #include <sys/stat.h> #include <sys/types.h> #include <sys/vfs.h> @@ -99,6 +100,13 @@ static bool is_bpffs(char *path) return (unsigned long)st_fs.f_type == BPF_FS_MAGIC; } +void set_max_rlimit(void) +{ + struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; + + setrlimit(RLIMIT_MEMLOCK, &rinf); +} + static int mnt_bpffs(const char *target, char *buff, size_t bufflen) { bool bind_done = false; @@ -169,34 +177,23 @@ int open_obj_pinned_any(char *path, enum bpf_obj_type exp_type) return fd; } -int do_pin_fd(int fd, const char *name) +int mount_bpffs_for_pin(const char *name) { char err_str[ERR_MAX_LEN]; char *file; char *dir; int err = 0; - err = bpf_obj_pin(fd, name); - if (!err) - goto out; - file = malloc(strlen(name) + 1); strcpy(file, name); dir = dirname(file); - if (errno != EPERM || is_bpffs(dir)) { - p_err("can't pin the object (%s): %s", name, strerror(errno)); + if (is_bpffs(dir)) + /* nothing to do if already mounted */ goto out_free; - } - /* Attempt to mount bpffs, then retry pinning. */ err = mnt_bpffs(dir, err_str, ERR_MAX_LEN); - if (!err) { - err = bpf_obj_pin(fd, name); - if (err) - p_err("can't pin the object (%s): %s", name, - strerror(errno)); - } else { + if (err) { err_str[ERR_MAX_LEN - 1] = '\0'; p_err("can't mount BPF file system to pin the object (%s): %s", name, err_str); @@ -204,10 +201,20 @@ int do_pin_fd(int fd, const char *name) out_free: free(file); -out: return err; } +int do_pin_fd(int fd, const char *name) +{ + int err; + + err = mount_bpffs_for_pin(name); + if (err) + return err; + + return bpf_obj_pin(fd, name); +} + int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(__u32)) { unsigned int id; diff --git a/tools/bpf/bpftool/jit_disasm.c b/tools/bpf/bpftool/jit_disasm.c index c75ffd9ce2bb..b2ed5ee1af5f 100644 --- a/tools/bpf/bpftool/jit_disasm.c +++ b/tools/bpf/bpftool/jit_disasm.c @@ -109,7 +109,7 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, if (inf) { bfdf->arch_info = inf; } else { - p_err("No libfd support for %s", arch); + p_err("No libbfd support for %s", arch); return; } } @@ -183,3 +183,9 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, bfd_close(bfdf); } + +int disasm_init(void) +{ + bfd_init(); + return 0; +} diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 75a3296dc0bc..5c4c1cd5a7ba 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -31,7 +31,6 @@ * SOFTWARE. */ -#include <bfd.h> #include <ctype.h> #include <errno.h> #include <getopt.h> @@ -399,8 +398,6 @@ int main(int argc, char **argv) if (argc < 0) usage(); - bfd_init(); - ret = cmd_select(cmds, argc, argv, do_help); if (json_output) diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 28322ace2856..3e8979567cf1 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -100,6 +100,8 @@ bool is_prefix(const char *pfx, const char *str); void fprint_hex(FILE *f, void *arg, unsigned int n, const char *sep); void usage(void) __noreturn; +void set_max_rlimit(void); + struct pinned_obj_table { DECLARE_HASHTABLE(table, 16); }; @@ -129,6 +131,7 @@ const char *get_fd_type_name(enum bpf_obj_type type); char *get_fdinfo(int fd, const char *key); int open_obj_pinned(char *path); int open_obj_pinned_any(char *path, enum bpf_obj_type exp_type); +int mount_bpffs_for_pin(const char *name); int do_pin_any(int argc, char **argv, int (*get_fd_by_id)(__u32)); int do_pin_fd(int fd, const char *name); @@ -144,8 +147,22 @@ int prog_parse_fd(int *argc, char ***argv); int map_parse_fd(int *argc, char ***argv); int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len); +#ifdef HAVE_LIBBFD_SUPPORT void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, const char *arch, const char *disassembler_options); +int disasm_init(void); +#else +static inline +void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, + const char *arch, const char *disassembler_options) +{ +} +static inline int disasm_init(void) +{ + p_err("No libbfd support"); + return -1; +} +#endif void print_data_json(uint8_t *data, size_t len); void print_hex_data_json(uint8_t *data, size_t len); @@ -170,6 +187,8 @@ struct btf_dumper { */ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id, const void *data); +void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id, + char *func_only, int size); struct nlattr; struct ifinfomsg; diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 7bf38f0e152e..cbd3080e72c7 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -52,28 +52,30 @@ #include "main.h" static const char * const map_type_name[] = { - [BPF_MAP_TYPE_UNSPEC] = "unspec", - [BPF_MAP_TYPE_HASH] = "hash", - [BPF_MAP_TYPE_ARRAY] = "array", - [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", - [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", - [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", - [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", - [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", - [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", - [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", - [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", - [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", - [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", - [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", - [BPF_MAP_TYPE_DEVMAP] = "devmap", - [BPF_MAP_TYPE_SOCKMAP] = "sockmap", - [BPF_MAP_TYPE_CPUMAP] = "cpumap", - [BPF_MAP_TYPE_XSKMAP] = "xskmap", - [BPF_MAP_TYPE_SOCKHASH] = "sockhash", - [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", - [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", + [BPF_MAP_TYPE_UNSPEC] = "unspec", + [BPF_MAP_TYPE_HASH] = "hash", + [BPF_MAP_TYPE_ARRAY] = "array", + [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", + [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", + [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", + [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", + [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", + [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", + [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", + [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", + [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", + [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", + [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", + [BPF_MAP_TYPE_DEVMAP] = "devmap", + [BPF_MAP_TYPE_SOCKMAP] = "sockmap", + [BPF_MAP_TYPE_CPUMAP] = "cpumap", + [BPF_MAP_TYPE_XSKMAP] = "xskmap", + [BPF_MAP_TYPE_SOCKHASH] = "sockhash", + [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", + [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", + [BPF_MAP_TYPE_QUEUE] = "queue", + [BPF_MAP_TYPE_STACK] = "stack", }; static bool map_is_per_cpu(__u32 type) @@ -215,70 +217,6 @@ err_end_obj: return ret; } -static int get_btf(struct bpf_map_info *map_info, struct btf **btf) -{ - struct bpf_btf_info btf_info = { 0 }; - __u32 len = sizeof(btf_info); - __u32 last_size; - int btf_fd; - void *ptr; - int err; - - err = 0; - *btf = NULL; - btf_fd = bpf_btf_get_fd_by_id(map_info->btf_id); - if (btf_fd < 0) - return 0; - - /* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so - * let's start with a sane default - 4KiB here - and resize it only if - * bpf_obj_get_info_by_fd() needs a bigger buffer. - */ - btf_info.btf_size = 4096; - last_size = btf_info.btf_size; - ptr = malloc(last_size); - if (!ptr) { - err = -ENOMEM; - goto exit_free; - } - - bzero(ptr, last_size); - btf_info.btf = ptr_to_u64(ptr); - err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); - - if (!err && btf_info.btf_size > last_size) { - void *temp_ptr; - - last_size = btf_info.btf_size; - temp_ptr = realloc(ptr, last_size); - if (!temp_ptr) { - err = -ENOMEM; - goto exit_free; - } - ptr = temp_ptr; - bzero(ptr, last_size); - btf_info.btf = ptr_to_u64(ptr); - err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); - } - - if (err || btf_info.btf_size > last_size) { - err = errno; - goto exit_free; - } - - *btf = btf__new((__u8 *)btf_info.btf, btf_info.btf_size, NULL); - if (IS_ERR(*btf)) { - err = PTR_ERR(*btf); - *btf = NULL; - } - -exit_free: - close(btf_fd); - free(ptr); - - return err; -} - static json_writer_t *get_btf_writer(void) { json_writer_t *jw = jsonw_new(stdout); @@ -383,7 +321,10 @@ static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, printf(single_line ? " " : "\n"); printf("value:%c", break_names ? '\n' : ' '); - fprint_hex(stdout, value, info->value_size, " "); + if (value) + fprint_hex(stdout, value, info->value_size, " "); + else + printf("<no entry>"); printf("\n"); } else { @@ -398,8 +339,11 @@ static void print_entry_plain(struct bpf_map_info *info, unsigned char *key, for (i = 0; i < n; i++) { printf("value (CPU %02d):%c", i, info->value_size > 16 ? '\n' : ' '); - fprint_hex(stdout, value + i * step, - info->value_size, " "); + if (value) + fprint_hex(stdout, value + i * step, + info->value_size, " "); + else + printf("<no entry>"); printf("\n"); } } @@ -731,7 +675,11 @@ static int dump_map_elem(int fd, void *key, void *value, jsonw_string_field(json_wtr, "error", strerror(lookup_errno)); jsonw_end_object(json_wtr); } else { - print_entry_error(map_info, key, strerror(lookup_errno)); + if (errno == ENOENT) + print_entry_plain(map_info, key, NULL); + else + print_entry_error(map_info, key, + strerror(lookup_errno)); } return 0; @@ -765,7 +713,7 @@ static int do_dump(int argc, char **argv) prev_key = NULL; - err = get_btf(&info, &btf); + err = btf_get_from_id(info.btf_id, &btf); if (err) { p_err("failed to get btf"); goto exit_free; @@ -909,7 +857,7 @@ static int do_lookup(int argc, char **argv) } /* here means bpf_map_lookup_elem() succeeded */ - err = get_btf(&info, &btf); + err = btf_get_from_id(info.btf_id, &btf); if (err) { p_err("failed to get btf"); goto exit_free; @@ -1140,6 +1088,8 @@ static int do_create(int argc, char **argv) return -1; } + set_max_rlimit(); + fd = bpf_create_map_xattr(&attr); if (fd < 0) { p_err("map create failed: %s", strerror(errno)); diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 5302ee282409..37b1daf19da6 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -47,6 +47,7 @@ #include <linux/err.h> #include <bpf.h> +#include <btf.h> #include <libbpf.h> #include "cfg.h" @@ -81,6 +82,7 @@ static const char * const attach_type_strings[] = { [BPF_SK_SKB_STREAM_PARSER] = "stream_parser", [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict", [BPF_SK_MSG_VERDICT] = "msg_verdict", + [BPF_FLOW_DISSECTOR] = "flow_dissector", [__MAX_BPF_ATTACH_TYPE] = NULL, }; @@ -450,14 +452,19 @@ static int do_dump(int argc, char **argv) struct bpf_prog_info info = {}; unsigned int *func_lens = NULL; const char *disasm_opt = NULL; + unsigned int finfo_rec_size; unsigned int nr_func_ksyms; unsigned int nr_func_lens; struct dump_data dd = {}; __u32 len = sizeof(info); + struct btf *btf = NULL; + void *func_info = NULL; + unsigned int finfo_cnt; unsigned int buf_size; char *filepath = NULL; bool opcodes = false; bool visual = false; + char func_sig[1024]; unsigned char *buf; __u32 *member_len; __u64 *member_ptr; @@ -466,6 +473,9 @@ static int do_dump(int argc, char **argv) int fd; if (is_prefix(*argv, "jited")) { + if (disasm_init()) + return -1; + member_len = &info.jited_prog_len; member_ptr = &info.jited_prog_insns; } else if (is_prefix(*argv, "xlated")) { @@ -547,6 +557,17 @@ static int do_dump(int argc, char **argv) } } + finfo_cnt = info.func_info_cnt; + finfo_rec_size = info.func_info_rec_size; + if (finfo_cnt && finfo_rec_size) { + func_info = malloc(finfo_cnt * finfo_rec_size); + if (!func_info) { + p_err("mem alloc failed"); + close(fd); + goto err_free; + } + } + memset(&info, 0, sizeof(info)); *member_ptr = ptr_to_u64(buf); @@ -555,6 +576,9 @@ static int do_dump(int argc, char **argv) info.nr_jited_ksyms = nr_func_ksyms; info.jited_func_lens = ptr_to_u64(func_lens); info.nr_jited_func_lens = nr_func_lens; + info.func_info_cnt = finfo_cnt; + info.func_info_rec_size = finfo_rec_size; + info.func_info = ptr_to_u64(func_info); err = bpf_obj_get_info_by_fd(fd, &info, &len); close(fd); @@ -578,6 +602,18 @@ static int do_dump(int argc, char **argv) goto err_free; } + if (info.func_info_cnt != finfo_cnt) { + p_err("incorrect func_info_cnt %d vs. expected %d", + info.func_info_cnt, finfo_cnt); + goto err_free; + } + + if (info.func_info_rec_size != finfo_rec_size) { + p_err("incorrect func_info_rec_size %d vs. expected %d", + info.func_info_rec_size, finfo_rec_size); + goto err_free; + } + if ((member_len == &info.jited_prog_len && info.jited_prog_insns == 0) || (member_len == &info.xlated_prog_len && @@ -586,6 +622,11 @@ static int do_dump(int argc, char **argv) goto err_free; } + if (info.btf_id && btf_get_from_id(info.btf_id, &btf)) { + p_err("failed to get btf"); + goto err_free; + } + if (filepath) { fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) { @@ -618,6 +659,7 @@ static int do_dump(int argc, char **argv) if (info.nr_jited_func_lens && info.jited_func_lens) { struct kernel_sym *sym = NULL; + struct bpf_func_info *record; char sym_name[SYM_MAX_NAME]; unsigned char *img = buf; __u64 *ksyms = NULL; @@ -644,12 +686,25 @@ static int do_dump(int argc, char **argv) strcpy(sym_name, "unknown"); } + if (func_info) { + record = func_info + i * finfo_rec_size; + btf_dumper_type_only(btf, record->type_id, + func_sig, + sizeof(func_sig)); + } + if (json_output) { jsonw_start_object(json_wtr); + if (func_info && func_sig[0] != '\0') { + jsonw_name(json_wtr, "proto"); + jsonw_string(json_wtr, func_sig); + } jsonw_name(json_wtr, "name"); jsonw_string(json_wtr, sym_name); jsonw_name(json_wtr, "insns"); } else { + if (func_info && func_sig[0] != '\0') + printf("%s:\n", func_sig); printf("%s:\n", sym_name); } @@ -678,6 +733,9 @@ static int do_dump(int argc, char **argv) kernel_syms_load(&dd); dd.nr_jited_ksyms = info.nr_jited_ksyms; dd.jited_ksyms = (__u64 *) info.jited_ksyms; + dd.btf = btf; + dd.func_info = func_info; + dd.finfo_rec_size = finfo_rec_size; if (json_output) dump_xlated_json(&dd, buf, *member_len, opcodes); @@ -689,12 +747,14 @@ static int do_dump(int argc, char **argv) free(buf); free(func_ksyms); free(func_lens); + free(func_info); return 0; err_free: free(buf); free(func_ksyms); free(func_lens); + free(func_info); return -1; } @@ -721,30 +781,49 @@ int map_replace_compar(const void *p1, const void *p2) return a->idx - b->idx; } -static int do_attach(int argc, char **argv) +static int parse_attach_detach_args(int argc, char **argv, int *progfd, + enum bpf_attach_type *attach_type, + int *mapfd) { - enum bpf_attach_type attach_type; - int err, mapfd, progfd; - - if (!REQ_ARGS(5)) { - p_err("too few parameters for map attach"); + if (!REQ_ARGS(3)) return -EINVAL; - } - progfd = prog_parse_fd(&argc, &argv); - if (progfd < 0) - return progfd; + *progfd = prog_parse_fd(&argc, &argv); + if (*progfd < 0) + return *progfd; - attach_type = parse_attach_type(*argv); - if (attach_type == __MAX_BPF_ATTACH_TYPE) { - p_err("invalid attach type"); + *attach_type = parse_attach_type(*argv); + if (*attach_type == __MAX_BPF_ATTACH_TYPE) { + p_err("invalid attach/detach type"); return -EINVAL; } + + if (*attach_type == BPF_FLOW_DISSECTOR) { + *mapfd = -1; + return 0; + } + NEXT_ARG(); + if (!REQ_ARGS(2)) + return -EINVAL; - mapfd = map_parse_fd(&argc, &argv); - if (mapfd < 0) - return mapfd; + *mapfd = map_parse_fd(&argc, &argv); + if (*mapfd < 0) + return *mapfd; + + return 0; +} + +static int do_attach(int argc, char **argv) +{ + enum bpf_attach_type attach_type; + int err, progfd; + int mapfd; + + err = parse_attach_detach_args(argc, argv, + &progfd, &attach_type, &mapfd); + if (err) + return err; err = bpf_prog_attach(progfd, mapfd, attach_type, 0); if (err) { @@ -760,27 +839,13 @@ static int do_attach(int argc, char **argv) static int do_detach(int argc, char **argv) { enum bpf_attach_type attach_type; - int err, mapfd, progfd; - - if (!REQ_ARGS(5)) { - p_err("too few parameters for map detach"); - return -EINVAL; - } + int err, progfd; + int mapfd; - progfd = prog_parse_fd(&argc, &argv); - if (progfd < 0) - return progfd; - - attach_type = parse_attach_type(*argv); - if (attach_type == __MAX_BPF_ATTACH_TYPE) { - p_err("invalid attach type"); - return -EINVAL; - } - NEXT_ARG(); - - mapfd = map_parse_fd(&argc, &argv); - if (mapfd < 0) - return mapfd; + err = parse_attach_detach_args(argc, argv, + &progfd, &attach_type, &mapfd); + if (err) + return err; err = bpf_prog_detach2(progfd, mapfd, attach_type); if (err) { @@ -792,15 +857,17 @@ static int do_detach(int argc, char **argv) jsonw_null(json_wtr); return 0; } -static int do_load(int argc, char **argv) + +static int load_with_options(int argc, char **argv, bool first_prog_only) { enum bpf_attach_type expected_attach_type; struct bpf_object_open_attr attr = { .prog_type = BPF_PROG_TYPE_UNSPEC, }; struct map_replace *map_replace = NULL; + struct bpf_program *prog = NULL, *pos; unsigned int old_map_fds = 0; - struct bpf_program *prog; + const char *pinmaps = NULL; struct bpf_object *obj; struct bpf_map *map; const char *pinfile; @@ -905,6 +972,13 @@ static int do_load(int argc, char **argv) goto err_free_reuse_maps; } NEXT_ARG(); + } else if (is_prefix(*argv, "pinmaps")) { + NEXT_ARG(); + + if (!REQ_ARGS(1)) + goto err_free_reuse_maps; + + pinmaps = GET_ARG(); } else { p_err("expected no more arguments, 'type', 'map' or 'dev', got: '%s'?", *argv); @@ -918,26 +992,25 @@ static int do_load(int argc, char **argv) goto err_free_reuse_maps; } - prog = bpf_program__next(NULL, obj); - if (!prog) { - p_err("object file doesn't contain any bpf program"); - goto err_close_obj; - } + bpf_object__for_each_program(pos, obj) { + enum bpf_prog_type prog_type = attr.prog_type; - bpf_program__set_ifindex(prog, ifindex); - if (attr.prog_type == BPF_PROG_TYPE_UNSPEC) { - const char *sec_name = bpf_program__title(prog, false); + if (attr.prog_type == BPF_PROG_TYPE_UNSPEC) { + const char *sec_name = bpf_program__title(pos, false); - err = libbpf_prog_type_by_name(sec_name, &attr.prog_type, - &expected_attach_type); - if (err < 0) { - p_err("failed to guess program type based on section name %s\n", - sec_name); - goto err_close_obj; + err = libbpf_prog_type_by_name(sec_name, &prog_type, + &expected_attach_type); + if (err < 0) { + p_err("failed to guess program type based on section name %s\n", + sec_name); + goto err_close_obj; + } } + + bpf_program__set_ifindex(pos, ifindex); + bpf_program__set_type(pos, prog_type); + bpf_program__set_expected_attach_type(pos, expected_attach_type); } - bpf_program__set_type(prog, attr.prog_type); - bpf_program__set_expected_attach_type(prog, expected_attach_type); qsort(map_replace, old_map_fds, sizeof(*map_replace), map_replace_compar); @@ -995,15 +1068,47 @@ static int do_load(int argc, char **argv) goto err_close_obj; } + set_max_rlimit(); + err = bpf_object__load(obj); if (err) { p_err("failed to load object file"); goto err_close_obj; } - if (do_pin_fd(bpf_program__fd(prog), pinfile)) + err = mount_bpffs_for_pin(pinfile); + if (err) goto err_close_obj; + if (first_prog_only) { + prog = bpf_program__next(NULL, obj); + if (!prog) { + p_err("object file doesn't contain any bpf program"); + goto err_close_obj; + } + + err = bpf_obj_pin(bpf_program__fd(prog), pinfile); + if (err) { + p_err("failed to pin program %s", + bpf_program__title(prog, false)); + goto err_close_obj; + } + } else { + err = bpf_object__pin_programs(obj, pinfile); + if (err) { + p_err("failed to pin all programs"); + goto err_close_obj; + } + } + + if (pinmaps) { + err = bpf_object__pin_maps(obj, pinmaps); + if (err) { + p_err("failed to pin all maps"); + goto err_unpin; + } + } + if (json_output) jsonw_null(json_wtr); @@ -1014,6 +1119,11 @@ static int do_load(int argc, char **argv) return 0; +err_unpin: + if (first_prog_only) + unlink(pinfile); + else + bpf_object__unpin_programs(obj, pinfile); err_close_obj: bpf_object__close(obj); err_free_reuse_maps: @@ -1023,6 +1133,16 @@ err_free_reuse_maps: return -1; } +static int do_load(int argc, char **argv) +{ + return load_with_options(argc, argv, true); +} + +static int do_loadall(int argc, char **argv) +{ + return load_with_options(argc, argv, false); +} + static int do_help(int argc, char **argv) { if (json_output) { @@ -1035,10 +1155,12 @@ static int do_help(int argc, char **argv) " %s %s dump xlated PROG [{ file FILE | opcodes | visual }]\n" " %s %s dump jited PROG [{ file FILE | opcodes }]\n" " %s %s pin PROG FILE\n" - " %s %s load OBJ FILE [type TYPE] [dev NAME] \\\n" - " [map { idx IDX | name NAME } MAP]\n" - " %s %s attach PROG ATTACH_TYPE MAP\n" - " %s %s detach PROG ATTACH_TYPE MAP\n" + " %s %s { load | loadall } OBJ PATH \\\n" + " [type TYPE] [dev NAME] \\\n" + " [map { idx IDX | name NAME } MAP]\\\n" + " [pinmaps MAP_DIR]\n" + " %s %s attach PROG ATTACH_TYPE [MAP]\n" + " %s %s detach PROG ATTACH_TYPE [MAP]\n" " %s %s help\n" "\n" " " HELP_SPEC_MAP "\n" @@ -1050,7 +1172,8 @@ static int do_help(int argc, char **argv) " cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n" " cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n" " cgroup/sendmsg4 | cgroup/sendmsg6 }\n" - " ATTACH_TYPE := { msg_verdict | skb_verdict | skb_parse }\n" + " ATTACH_TYPE := { msg_verdict | skb_verdict | skb_parse |\n" + " flow_dissector }\n" " " HELP_SPEC_OPTIONS "\n" "", bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], @@ -1067,6 +1190,7 @@ static const struct cmd cmds[] = { { "dump", do_dump }, { "pin", do_pin }, { "load", do_load }, + { "loadall", do_loadall }, { "attach", do_attach }, { "detach", do_detach }, { 0 } diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c index 3284759df98a..e06ac0286a75 100644 --- a/tools/bpf/bpftool/xlated_dumper.c +++ b/tools/bpf/bpftool/xlated_dumper.c @@ -242,11 +242,15 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, .cb_imm = print_imm, .private_data = dd, }; + struct bpf_func_info *record; struct bpf_insn *insn = buf; + struct btf *btf = dd->btf; bool double_insn = false; + char func_sig[1024]; unsigned int i; jsonw_start_array(json_wtr); + record = dd->func_info; for (i = 0; i < len / sizeof(*insn); i++) { if (double_insn) { double_insn = false; @@ -255,6 +259,20 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW); jsonw_start_object(json_wtr); + + if (btf && record) { + if (record->insn_offset == i) { + btf_dumper_type_only(btf, record->type_id, + func_sig, + sizeof(func_sig)); + if (func_sig[0] != '\0') { + jsonw_name(json_wtr, "proto"); + jsonw_string(json_wtr, func_sig); + } + record = (void *)record + dd->finfo_rec_size; + } + } + jsonw_name(json_wtr, "disasm"); print_bpf_insn(&cbs, insn + i, true); @@ -297,16 +315,31 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, .cb_imm = print_imm, .private_data = dd, }; + struct bpf_func_info *record; struct bpf_insn *insn = buf; + struct btf *btf = dd->btf; bool double_insn = false; + char func_sig[1024]; unsigned int i; + record = dd->func_info; for (i = 0; i < len / sizeof(*insn); i++) { if (double_insn) { double_insn = false; continue; } + if (btf && record) { + if (record->insn_offset == i) { + btf_dumper_type_only(btf, record->type_id, + func_sig, + sizeof(func_sig)); + if (func_sig[0] != '\0') + printf("%s:\n", func_sig); + record = (void *)record + dd->finfo_rec_size; + } + } + double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW); printf("% 4d: ", i); diff --git a/tools/bpf/bpftool/xlated_dumper.h b/tools/bpf/bpftool/xlated_dumper.h index 33d86e2b369b..aec31723e1e5 100644 --- a/tools/bpf/bpftool/xlated_dumper.h +++ b/tools/bpf/bpftool/xlated_dumper.h @@ -51,6 +51,9 @@ struct dump_data { __u32 sym_count; __u64 *jited_ksyms; __u32 nr_jited_ksyms; + struct btf *btf; + void *func_info; + __u32 finfo_rec_size; char scratch_buff[SYM_MAX_NAME + 8]; }; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 852dc17ab47a..23e2031a43d4 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -257,9 +257,6 @@ enum bpf_attach_type { /* Specify numa node during map creation */ #define BPF_F_NUMA_NODE (1U << 2) -/* flags for BPF_PROG_QUERY */ -#define BPF_F_QUERY_EFFECTIVE (1U << 0) - #define BPF_OBJ_NAME_LEN 16U /* Flags for accessing BPF object */ @@ -269,6 +266,12 @@ enum bpf_attach_type { /* Flag for stack_map, store build_id+offset instead of pointer */ #define BPF_F_STACK_BUILD_ID (1U << 5) +/* Zero-initialize hash function seed. This should only be used for testing. */ +#define BPF_F_ZERO_SEED (1U << 6) + +/* flags for BPF_PROG_QUERY */ +#define BPF_F_QUERY_EFFECTIVE (1U << 0) + enum bpf_stack_build_id_status { /* user space need an empty entry to identify end of a trace */ BPF_STACK_BUILD_ID_EMPTY = 0, @@ -335,6 +338,10 @@ union bpf_attr { * (context accesses, allowed helpers, etc). */ __u32 expected_attach_type; + __u32 prog_btf_fd; /* fd pointing to BTF type data */ + __u32 func_info_rec_size; /* userspace bpf_func_info size */ + __aligned_u64 func_info; /* func info */ + __u32 func_info_cnt; /* number of bpf_func_info records */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -2201,6 +2208,8 @@ union bpf_attr { * **CONFIG_NET** configuration option. * Return * Pointer to *struct bpf_sock*, or NULL in case of failure. + * For sockets with reuseport option, *struct bpf_sock* + * return is from reuse->socks[] using hash of the packet. * * struct bpf_sock *bpf_sk_lookup_udp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u32 netns, u64 flags) * Description @@ -2233,6 +2242,8 @@ union bpf_attr { * **CONFIG_NET** configuration option. * Return * Pointer to *struct bpf_sock*, or NULL in case of failure. + * For sockets with reuseport option, *struct bpf_sock* + * return is from reuse->socks[] using hash of the packet. * * int bpf_sk_release(struct bpf_sock *sk) * Description @@ -2457,6 +2468,7 @@ struct __sk_buff { __u32 data_meta; struct bpf_flow_keys *flow_keys; + __u64 tstamp; }; struct bpf_tunnel_key { @@ -2631,6 +2643,10 @@ struct bpf_prog_info { __u32 nr_jited_func_lens; __aligned_u64 jited_ksyms; __aligned_u64 jited_func_lens; + __u32 btf_id; + __u32 func_info_rec_size; + __aligned_u64 func_info; + __u32 func_info_cnt; } __attribute__((aligned(8))); struct bpf_map_info { @@ -2942,4 +2958,9 @@ struct bpf_flow_keys { }; }; +struct bpf_func_info { + __u32 insn_offset; + __u32 type_id; +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/tools/include/uapi/linux/btf.h b/tools/include/uapi/linux/btf.h index 972265f32871..14f66948fc95 100644 --- a/tools/include/uapi/linux/btf.h +++ b/tools/include/uapi/linux/btf.h @@ -40,7 +40,8 @@ struct btf_type { /* "size" is used by INT, ENUM, STRUCT and UNION. * "size" tells the size of the type it is describing. * - * "type" is used by PTR, TYPEDEF, VOLATILE, CONST and RESTRICT. + * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, + * FUNC and FUNC_PROTO. * "type" is a type_id referring to another type. */ union { @@ -64,8 +65,10 @@ struct btf_type { #define BTF_KIND_VOLATILE 9 /* Volatile */ #define BTF_KIND_CONST 10 /* Const */ #define BTF_KIND_RESTRICT 11 /* Restrict */ -#define BTF_KIND_MAX 11 -#define NR_BTF_KINDS 12 +#define BTF_KIND_FUNC 12 /* Function */ +#define BTF_KIND_FUNC_PROTO 13 /* Function Proto */ +#define BTF_KIND_MAX 13 +#define NR_BTF_KINDS 14 /* For some specific BTF_KIND, "struct btf_type" is immediately * followed by extra data. @@ -110,4 +113,13 @@ struct btf_member { __u32 offset; /* offset in bits */ }; +/* BTF_KIND_FUNC_PROTO is followed by multiple "struct btf_param". + * The exact number of btf_param is stored in the vlen (of the + * info in "struct btf_type"). + */ +struct btf_param { + __u32 name_off; + __u32 type; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index 425b480bda75..1b4a683a00fc 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -66,7 +66,7 @@ ifndef VERBOSE endif FEATURE_USER = .libbpf -FEATURE_TESTS = libelf libelf-mmap bpf reallocarray +FEATURE_TESTS = libelf libelf-mmap bpf reallocarray cxx FEATURE_DISPLAY = libelf bpf INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/arch/$(ARCH)/include/uapi -I$(srctree)/tools/include/uapi @@ -148,6 +148,12 @@ LIB_FILE := $(addprefix $(OUTPUT),$(LIB_FILE)) CMD_TARGETS = $(LIB_FILE) +CXX_TEST_TARGET = $(OUTPUT)test_libbpf + +ifeq ($(feature-cxx), 1) + CMD_TARGETS += $(CXX_TEST_TARGET) +endif + TARGETS = $(CMD_TARGETS) all: fixdep all_cmd @@ -175,6 +181,9 @@ $(OUTPUT)libbpf.so: $(BPF_IN) $(OUTPUT)libbpf.a: $(BPF_IN) $(QUIET_LINK)$(RM) $@; $(AR) rcs $@ $^ +$(OUTPUT)test_libbpf: test_libbpf.cpp $(OUTPUT)libbpf.a + $(QUIET_LINK)$(CXX) $^ -lelf -o $@ + define do_install if [ ! -d '$(DESTDIR_SQ)$2' ]; then \ $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$2'; \ @@ -201,8 +210,8 @@ config-clean: $(Q)$(MAKE) -C $(srctree)/tools/build/feature/ clean >/dev/null clean: - $(call QUIET_CLEAN, libbpf) $(RM) *.o *~ $(TARGETS) *.a *.so .*.d .*.cmd \ - $(RM) LIBBPF-CFLAGS + $(call QUIET_CLEAN, libbpf) $(RM) $(TARGETS) $(CXX_TEST_TARGET) \ + *.o *~ *.a *.so .*.d .*.cmd LIBBPF-CFLAGS $(call QUIET_CLEAN, core-gen) $(RM) $(OUTPUT)FEATURE-DUMP.libbpf diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 03f9bcc4ef50..ce1822194590 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -177,6 +177,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, char *log_buf, size_t log_buf_sz) { union bpf_attr attr; + void *finfo = NULL; __u32 name_len; int fd; @@ -196,6 +197,10 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, attr.log_level = 0; attr.kern_version = load_attr->kern_version; attr.prog_ifindex = load_attr->prog_ifindex; + attr.prog_btf_fd = load_attr->prog_btf_fd; + attr.func_info_rec_size = load_attr->func_info_rec_size; + attr.func_info_cnt = load_attr->func_info_cnt; + attr.func_info = ptr_to_u64(load_attr->func_info); memcpy(attr.prog_name, load_attr->name, min(name_len, BPF_OBJ_NAME_LEN - 1)); @@ -203,12 +208,55 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, if (fd >= 0 || !log_buf || !log_buf_sz) return fd; + /* After bpf_prog_load, the kernel may modify certain attributes + * to give user space a hint how to deal with loading failure. + * Check to see whether we can make some changes and load again. + */ + if (errno == E2BIG && attr.func_info_cnt && + attr.func_info_rec_size < load_attr->func_info_rec_size) { + __u32 actual_rec_size = load_attr->func_info_rec_size; + __u32 expected_rec_size = attr.func_info_rec_size; + __u32 finfo_cnt = load_attr->func_info_cnt; + __u64 finfo_len = actual_rec_size * finfo_cnt; + const void *orecord; + void *nrecord; + int i; + + finfo = malloc(finfo_len); + if (!finfo) + /* further try with log buffer won't help */ + return fd; + + /* zero out bytes kernel does not understand */ + orecord = load_attr->func_info; + nrecord = finfo; + for (i = 0; i < load_attr->func_info_cnt; i++) { + memcpy(nrecord, orecord, expected_rec_size); + memset(nrecord + expected_rec_size, 0, + actual_rec_size - expected_rec_size); + orecord += actual_rec_size; + nrecord += actual_rec_size; + } + + /* try with corrected func info records */ + attr.func_info = ptr_to_u64(finfo); + attr.func_info_rec_size = load_attr->func_info_rec_size; + + fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); + + if (fd >= 0 || !log_buf || !log_buf_sz) + goto done; + } + /* Try again with log */ attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_buf_sz; attr.log_level = 1; log_buf[0] = 0; - return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); + fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); +done: + free(finfo); + return fd; } int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 26a51538213c..09e8bbe111d4 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -27,6 +27,10 @@ #include <stdbool.h> #include <stddef.h> +#ifdef __cplusplus +extern "C" { +#endif + #ifndef LIBBPF_API #define LIBBPF_API __attribute__((visibility("default"))) #endif @@ -74,6 +78,10 @@ struct bpf_load_program_attr { const char *license; __u32 kern_version; __u32 prog_ifindex; + __u32 prog_btf_fd; + __u32 func_info_rec_size; + const void *func_info; + __u32 func_info_cnt; }; /* Flags to direct loading requirements */ @@ -128,4 +136,9 @@ LIBBPF_API int bpf_load_btf(void *btf, __u32 btf_size, char *log_buf, LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset, __u64 *probe_addr); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + #endif /* __LIBBPF_BPF_H */ diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 449591aa9900..13ddc4bd24ee 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -37,6 +37,23 @@ struct btf { int fd; }; +struct btf_ext { + void *func_info; + __u32 func_info_rec_size; + __u32 func_info_len; +}; + +/* The minimum bpf_func_info checked by the loader */ +struct bpf_func_info_min { + __u32 insn_offset; + __u32 type_id; +}; + +static inline __u64 ptr_to_u64(const void *ptr) +{ + return (__u64) (unsigned long) ptr; +} + static int btf_add_type(struct btf *btf, struct btf_type *t) { if (btf->types_size - btf->nr_types < 2) { @@ -165,6 +182,10 @@ static int btf_parse_type_sec(struct btf *btf, btf_print_fn_t err_log) case BTF_KIND_ENUM: next_type += vlen * sizeof(struct btf_enum); break; + case BTF_KIND_FUNC_PROTO: + next_type += vlen * sizeof(struct btf_param); + break; + case BTF_KIND_FUNC: case BTF_KIND_TYPEDEF: case BTF_KIND_PTR: case BTF_KIND_FWD: @@ -393,3 +414,329 @@ const char *btf__name_by_offset(const struct btf *btf, __u32 offset) else return NULL; } + +int btf_get_from_id(__u32 id, struct btf **btf) +{ + struct bpf_btf_info btf_info = { 0 }; + __u32 len = sizeof(btf_info); + __u32 last_size; + int btf_fd; + void *ptr; + int err; + + err = 0; + *btf = NULL; + btf_fd = bpf_btf_get_fd_by_id(id); + if (btf_fd < 0) + return 0; + + /* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so + * let's start with a sane default - 4KiB here - and resize it only if + * bpf_obj_get_info_by_fd() needs a bigger buffer. + */ + btf_info.btf_size = 4096; + last_size = btf_info.btf_size; + ptr = malloc(last_size); + if (!ptr) { + err = -ENOMEM; + goto exit_free; + } + + bzero(ptr, last_size); + btf_info.btf = ptr_to_u64(ptr); + err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); + + if (!err && btf_info.btf_size > last_size) { + void *temp_ptr; + + last_size = btf_info.btf_size; + temp_ptr = realloc(ptr, last_size); + if (!temp_ptr) { + err = -ENOMEM; + goto exit_free; + } + ptr = temp_ptr; + bzero(ptr, last_size); + btf_info.btf = ptr_to_u64(ptr); + err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); + } + + if (err || btf_info.btf_size > last_size) { + err = errno; + goto exit_free; + } + + *btf = btf__new((__u8 *)btf_info.btf, btf_info.btf_size, NULL); + if (IS_ERR(*btf)) { + err = PTR_ERR(*btf); + *btf = NULL; + } + +exit_free: + close(btf_fd); + free(ptr); + + return err; +} + +static int btf_ext_validate_func_info(const void *finfo, __u32 size, + btf_print_fn_t err_log) +{ + int sec_hdrlen = sizeof(struct btf_sec_func_info); + __u32 size_left, num_records, record_size; + const struct btf_sec_func_info *sinfo; + __u64 total_record_size; + + /* At least a func_info record size */ + if (size < sizeof(__u32)) { + elog("BTF.ext func_info record size not found"); + return -EINVAL; + } + + /* The record size needs to meet below minimum standard */ + record_size = *(__u32 *)finfo; + if (record_size < sizeof(struct bpf_func_info_min) || + record_size % sizeof(__u32)) { + elog("BTF.ext func_info invalid record size"); + return -EINVAL; + } + + sinfo = finfo + sizeof(__u32); + size_left = size - sizeof(__u32); + + /* If no func_info records, return failure now so .BTF.ext + * won't be used. + */ + if (!size_left) { + elog("BTF.ext no func info records"); + return -EINVAL; + } + + while (size_left) { + if (size_left < sec_hdrlen) { + elog("BTF.ext func_info header not found"); + return -EINVAL; + } + + num_records = sinfo->num_func_info; + if (num_records == 0) { + elog("incorrect BTF.ext num_func_info"); + return -EINVAL; + } + + total_record_size = sec_hdrlen + + (__u64)num_records * record_size; + if (size_left < total_record_size) { + elog("incorrect BTF.ext num_func_info"); + return -EINVAL; + } + + size_left -= total_record_size; + sinfo = (void *)sinfo + total_record_size; + } + + return 0; +} + +static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, + btf_print_fn_t err_log) +{ + const struct btf_ext_header *hdr = (struct btf_ext_header *)data; + __u32 meta_left, last_func_info_pos; + void *finfo; + + if (data_size < offsetof(struct btf_ext_header, func_info_off) || + data_size < hdr->hdr_len) { + elog("BTF.ext header not found"); + return -EINVAL; + } + + if (hdr->magic != BTF_MAGIC) { + elog("Invalid BTF.ext magic:%x\n", hdr->magic); + return -EINVAL; + } + + if (hdr->version != BTF_VERSION) { + elog("Unsupported BTF.ext version:%u\n", hdr->version); + return -ENOTSUP; + } + + if (hdr->flags) { + elog("Unsupported BTF.ext flags:%x\n", hdr->flags); + return -ENOTSUP; + } + + meta_left = data_size - hdr->hdr_len; + if (!meta_left) { + elog("BTF.ext has no data\n"); + return -EINVAL; + } + + if (meta_left < hdr->func_info_off) { + elog("Invalid BTF.ext func_info section offset:%u\n", + hdr->func_info_off); + return -EINVAL; + } + + if (hdr->func_info_off & 0x03) { + elog("BTF.ext func_info section is not aligned to 4 bytes\n"); + return -EINVAL; + } + + last_func_info_pos = hdr->hdr_len + hdr->func_info_off + + hdr->func_info_len; + if (last_func_info_pos > data_size) { + elog("Invalid BTF.ext func_info section size:%u\n", + hdr->func_info_len); + return -EINVAL; + } + + finfo = data + hdr->hdr_len + hdr->func_info_off; + return btf_ext_validate_func_info(finfo, hdr->func_info_len, + err_log); +} + +void btf_ext__free(struct btf_ext *btf_ext) +{ + if (!btf_ext) + return; + + free(btf_ext->func_info); + free(btf_ext); +} + +struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) +{ + const struct btf_ext_header *hdr; + struct btf_ext *btf_ext; + void *org_fdata, *fdata; + __u32 hdrlen, size_u32; + int err; + + err = btf_ext_parse_hdr(data, size, err_log); + if (err) + return ERR_PTR(err); + + btf_ext = calloc(1, sizeof(struct btf_ext)); + if (!btf_ext) + return ERR_PTR(-ENOMEM); + + hdr = (const struct btf_ext_header *)data; + hdrlen = hdr->hdr_len; + size_u32 = sizeof(__u32); + fdata = malloc(hdr->func_info_len - size_u32); + if (!fdata) { + free(btf_ext); + return ERR_PTR(-ENOMEM); + } + + /* remember record size and copy rest of func_info data */ + org_fdata = data + hdrlen + hdr->func_info_off; + btf_ext->func_info_rec_size = *(__u32 *)org_fdata; + memcpy(fdata, org_fdata + size_u32, hdr->func_info_len - size_u32); + btf_ext->func_info = fdata; + btf_ext->func_info_len = hdr->func_info_len - size_u32; + + return btf_ext; +} + +int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, + const char *sec_name, void **func_info, + __u32 *func_info_rec_size, __u32 *func_info_len) +{ + __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); + __u32 i, record_size, records_len; + struct btf_sec_func_info *sinfo; + const char *info_sec_name; + __s64 remain_len; + void *data; + + record_size = btf_ext->func_info_rec_size; + sinfo = btf_ext->func_info; + remain_len = btf_ext->func_info_len; + + while (remain_len > 0) { + records_len = sinfo->num_func_info * record_size; + info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); + if (strcmp(info_sec_name, sec_name)) { + remain_len -= sec_hdrlen + records_len; + sinfo = (void *)sinfo + sec_hdrlen + records_len; + continue; + } + + data = malloc(records_len); + if (!data) + return -ENOMEM; + + memcpy(data, sinfo->data, records_len); + + /* adjust the insn_offset, the data in .BTF.ext is + * the actual byte offset, and the kernel expects + * the offset in term of bpf_insn. + * + * adjust the insn offset only, the rest data will + * be passed to kernel. + */ + for (i = 0; i < sinfo->num_func_info; i++) { + struct bpf_func_info_min *record; + + record = data + i * record_size; + record->insn_offset /= sizeof(struct bpf_insn); + } + + *func_info = data; + *func_info_len = records_len; + *func_info_rec_size = record_size; + return 0; + } + + return -EINVAL; +} + +int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **func_info, __u32 *func_info_len) +{ + __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); + __u32 i, record_size, existing_flen, records_len; + struct btf_sec_func_info *sinfo; + const char *info_sec_name; + __u64 remain_len; + void *data; + + record_size = btf_ext->func_info_rec_size; + sinfo = btf_ext->func_info; + remain_len = btf_ext->func_info_len; + while (remain_len > 0) { + records_len = sinfo->num_func_info * record_size; + info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); + if (strcmp(info_sec_name, sec_name)) { + remain_len -= sec_hdrlen + records_len; + sinfo = (void *)sinfo + sec_hdrlen + records_len; + continue; + } + + existing_flen = *func_info_len; + data = realloc(*func_info, existing_flen + records_len); + if (!data) + return -ENOMEM; + + memcpy(data + existing_flen, sinfo->data, records_len); + /* adjust insn_offset only, the rest data will be passed + * to the kernel. + */ + for (i = 0; i < sinfo->num_func_info; i++) { + struct bpf_func_info_min *record; + + record = data + existing_flen + i * record_size; + record->insn_offset = + record->insn_offset / sizeof(struct bpf_insn) + + insns_cnt; + } + *func_info = data; + *func_info_len = existing_flen + records_len; + return 0; + } + + return -EINVAL; +} diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index b77e7080f7e7..701ad2b6c41f 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -6,15 +6,60 @@ #include <linux/types.h> +#ifdef __cplusplus +extern "C" { +#endif + #ifndef LIBBPF_API #define LIBBPF_API __attribute__((visibility("default"))) #endif #define BTF_ELF_SEC ".BTF" +#define BTF_EXT_ELF_SEC ".BTF.ext" struct btf; +struct btf_ext; struct btf_type; +/* + * The .BTF.ext ELF section layout defined as + * struct btf_ext_header + * func_info subsection + * + * The func_info subsection layout: + * record size for struct bpf_func_info in the func_info subsection + * struct btf_sec_func_info for section #1 + * a list of bpf_func_info records for section #1 + * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h + * but may not be identical + * struct btf_sec_func_info for section #2 + * a list of bpf_func_info records for section #2 + * ...... + * + * Note that the bpf_func_info record size in .BTF.ext may not + * be the same as the one defined in include/uapi/linux/bpf.h. + * The loader should ensure that record_size meets minimum + * requirement and pass the record as is to the kernel. The + * kernel will handle the func_info properly based on its contents. + */ +struct btf_ext_header { + __u16 magic; + __u8 version; + __u8 flags; + __u32 hdr_len; + + /* All offsets are in bytes relative to the end of this header */ + __u32 func_info_off; + __u32 func_info_len; +}; + +struct btf_sec_func_info { + __u32 sec_name_off; + __u32 num_func_info; + /* Followed by num_func_info number of bpf func_info records */ + __u8 data[0]; +}; + typedef int (*btf_print_fn_t)(const char *, ...) __attribute__((format(printf, 1, 2))); @@ -28,5 +73,19 @@ LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id); LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id); LIBBPF_API int btf__fd(const struct btf *btf); LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset); +LIBBPF_API int btf_get_from_id(__u32 id, struct btf **btf); + +struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log); +void btf_ext__free(struct btf_ext *btf_ext); +int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, + const char *sec_name, void **func_info, + __u32 *func_info_rec_size, __u32 *func_info_len); +int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, void **func_info, + __u32 *func_info_len); + +#ifdef __cplusplus +} /* extern "C" */ +#endif #endif /* __LIBBPF_BTF_H */ diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index d6e62e90e8d4..ed4212a4c5f9 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -24,6 +24,7 @@ #include <linux/kernel.h> #include <linux/bpf.h> #include <linux/btf.h> +#include <linux/filter.h> #include <linux/list.h> #include <linux/limits.h> #include <linux/perf_event.h> @@ -114,6 +115,11 @@ void libbpf_set_print(libbpf_print_fn_t warn, # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ #endif +struct bpf_capabilities { + /* v4.14: kernel support for program & map names. */ + __u32 name:1; +}; + /* * bpf_prog should be a better name but it has been used in * linux/filter.h. @@ -124,6 +130,10 @@ struct bpf_program { char *name; int prog_ifindex; char *section_name; + /* section_name with / replaced by _; makes recursive pinning + * in bpf_object__pin_programs easier + */ + char *pin_name; struct bpf_insn *insns; size_t insns_cnt, main_prog_cnt; enum bpf_prog_type type; @@ -152,6 +162,12 @@ struct bpf_program { bpf_program_clear_priv_t clear_priv; enum bpf_attach_type expected_attach_type; + int btf_fd; + void *func_info; + __u32 func_info_rec_size; + __u32 func_info_len; + + struct bpf_capabilities *caps; }; struct bpf_map { @@ -159,6 +175,7 @@ struct bpf_map { char *name; size_t offset; int map_ifindex; + int inner_map_fd; struct bpf_map_def def; __u32 btf_key_type_id; __u32 btf_value_type_id; @@ -208,10 +225,13 @@ struct bpf_object { struct list_head list; struct btf *btf; + struct btf_ext *btf_ext; void *priv; bpf_object_clear_priv_t clear_priv; + struct bpf_capabilities caps; + char path[]; }; #define obj_elf_valid(o) ((o)->efile.elf) @@ -237,6 +257,9 @@ void bpf_program__unload(struct bpf_program *prog) prog->instances.nr = -1; zfree(&prog->instances.fds); + + zclose(prog->btf_fd); + zfree(&prog->func_info); } static void bpf_program__exit(struct bpf_program *prog) @@ -253,6 +276,7 @@ static void bpf_program__exit(struct bpf_program *prog) bpf_program__unload(prog); zfree(&prog->name); zfree(&prog->section_name); + zfree(&prog->pin_name); zfree(&prog->insns); zfree(&prog->reloc_desc); @@ -261,6 +285,17 @@ static void bpf_program__exit(struct bpf_program *prog) prog->idx = -1; } +static char *__bpf_program__pin_name(struct bpf_program *prog) +{ + char *name, *p; + + name = p = strdup(prog->section_name); + while ((p = strchr(p, '/'))) + *p = '_'; + + return name; +} + static int bpf_program__init(void *data, size_t size, char *section_name, int idx, struct bpf_program *prog) @@ -279,6 +314,13 @@ bpf_program__init(void *data, size_t size, char *section_name, int idx, goto errout; } + prog->pin_name = __bpf_program__pin_name(prog); + if (!prog->pin_name) { + pr_warning("failed to alloc pin name for prog under section(%d) %s\n", + idx, section_name); + goto errout; + } + prog->insns = malloc(size); if (!prog->insns) { pr_warning("failed to alloc insns for prog under section %s\n", @@ -291,7 +333,8 @@ bpf_program__init(void *data, size_t size, char *section_name, int idx, prog->idx = idx; prog->instances.fds = NULL; prog->instances.nr = -1; - prog->type = BPF_PROG_TYPE_KPROBE; + prog->type = BPF_PROG_TYPE_UNSPEC; + prog->btf_fd = -1; return 0; errout: @@ -310,6 +353,7 @@ bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, if (err) return err; + prog.caps = &obj->caps; progs = obj->programs; nr_progs = obj->nr_programs; @@ -562,6 +606,14 @@ static int compare_bpf_map(const void *_a, const void *_b) return a->offset - b->offset; } +static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) +{ + if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || + type == BPF_MAP_TYPE_HASH_OF_MAPS) + return true; + return false; +} + static int bpf_object__init_maps(struct bpf_object *obj, int flags) { @@ -625,13 +677,15 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) } obj->nr_maps = nr_maps; - /* - * fill all fd with -1 so won't close incorrect - * fd (fd=0 is stdin) when failure (zclose won't close - * negative fd)). - */ - for (i = 0; i < nr_maps; i++) + for (i = 0; i < nr_maps; i++) { + /* + * fill all fd with -1 so won't close incorrect + * fd (fd=0 is stdin) when failure (zclose won't close + * negative fd)). + */ obj->maps[i].fd = -1; + obj->maps[i].inner_map_fd = -1; + } /* * Fill obj->maps using data in "maps" section. @@ -784,6 +838,15 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) BTF_ELF_SEC, PTR_ERR(obj->btf)); obj->btf = NULL; } + } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { + obj->btf_ext = btf_ext__new(data->d_buf, data->d_size, + __pr_debug); + if (IS_ERR(obj->btf_ext)) { + pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", + BTF_EXT_ELF_SEC, + PTR_ERR(obj->btf_ext)); + obj->btf_ext = NULL; + } } else if (sh.sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warning("bpf: multiple SYMTAB in %s\n", @@ -1095,6 +1158,52 @@ err_free_new_name: } static int +bpf_object__probe_name(struct bpf_object *obj) +{ + struct bpf_load_program_attr attr; + char *cp, errmsg[STRERR_BUFSIZE]; + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + int ret; + + /* make sure basic loading works */ + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; + attr.insns = insns; + attr.insns_cnt = ARRAY_SIZE(insns); + attr.license = "GPL"; + + ret = bpf_load_program_xattr(&attr, NULL, 0); + if (ret < 0) { + cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); + pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", + __func__, cp, errno); + return -errno; + } + close(ret); + + /* now try the same program, but with the name */ + + attr.name = "test"; + ret = bpf_load_program_xattr(&attr, NULL, 0); + if (ret >= 0) { + obj->caps.name = 1; + close(ret); + } + + return 0; +} + +static int +bpf_object__probe_caps(struct bpf_object *obj) +{ + return bpf_object__probe_name(obj); +} + +static int bpf_object__create_maps(struct bpf_object *obj) { struct bpf_create_map_attr create_attr = {}; @@ -1113,7 +1222,8 @@ bpf_object__create_maps(struct bpf_object *obj) continue; } - create_attr.name = map->name; + if (obj->caps.name) + create_attr.name = map->name; create_attr.map_ifindex = map->map_ifindex; create_attr.map_type = def->type; create_attr.map_flags = def->map_flags; @@ -1123,6 +1233,9 @@ bpf_object__create_maps(struct bpf_object *obj) create_attr.btf_fd = 0; create_attr.btf_key_type_id = 0; create_attr.btf_value_type_id = 0; + if (bpf_map_type__is_map_in_map(def->type) && + map->inner_map_fd >= 0) + create_attr.inner_map_fd = map->inner_map_fd; if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { create_attr.btf_fd = btf__fd(obj->btf); @@ -1167,6 +1280,7 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, struct bpf_insn *insn, *new_insn; struct bpf_program *text; size_t new_cnt; + int err; if (relo->type != RELO_CALL) return -LIBBPF_ERRNO__RELOC; @@ -1189,6 +1303,20 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, pr_warning("oom in prog realloc\n"); return -ENOMEM; } + + if (obj->btf && obj->btf_ext) { + err = btf_ext__reloc(obj->btf, obj->btf_ext, + text->section_name, + prog->insns_cnt, + &prog->func_info, + &prog->func_info_len); + if (err) { + pr_warning("error in btf_ext__reloc for sec %s\n", + text->section_name); + return err; + } + } + memcpy(new_insn + prog->insns_cnt, text->insns, text->insns_cnt * sizeof(*insn)); prog->insns = new_insn; @@ -1208,7 +1336,24 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) { int i, err; - if (!prog || !prog->reloc_desc) + if (!prog) + return 0; + + if (obj->btf && obj->btf_ext) { + err = btf_ext__reloc_init(obj->btf, obj->btf_ext, + prog->section_name, + &prog->func_info, + &prog->func_info_rec_size, + &prog->func_info_len); + if (err) { + pr_warning("err in btf_ext__reloc_init for sec %s\n", + prog->section_name); + return err; + } + prog->btf_fd = btf__fd(obj->btf); + } + + if (!prog->reloc_desc) return 0; for (i = 0; i < prog->nr_reloc; i++) { @@ -1296,9 +1441,9 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) } static int -load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type, - const char *name, struct bpf_insn *insns, int insns_cnt, - char *license, __u32 kern_version, int *pfd, int prog_ifindex) +load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, + char *license, __u32 kern_version, int *pfd, + __u32 func_info_cnt) { struct bpf_load_program_attr load_attr; char *cp, errmsg[STRERR_BUFSIZE]; @@ -1306,14 +1451,19 @@ load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type, int ret; memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); - load_attr.prog_type = type; - load_attr.expected_attach_type = expected_attach_type; - load_attr.name = name; + load_attr.prog_type = prog->type; + load_attr.expected_attach_type = prog->expected_attach_type; + if (prog->caps->name) + load_attr.name = prog->name; load_attr.insns = insns; load_attr.insns_cnt = insns_cnt; load_attr.license = license; load_attr.kern_version = kern_version; - load_attr.prog_ifindex = prog_ifindex; + load_attr.prog_ifindex = prog->prog_ifindex; + load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; + load_attr.func_info = prog->func_info; + load_attr.func_info_rec_size = prog->func_info_rec_size; + load_attr.func_info_cnt = func_info_cnt; if (!load_attr.insns || !load_attr.insns_cnt) return -EINVAL; @@ -1371,8 +1521,14 @@ int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_version) { + __u32 func_info_cnt; int err = 0, fd, i; + if (prog->func_info_len == 0) + func_info_cnt = 0; + else + func_info_cnt = prog->func_info_len / prog->func_info_rec_size; + if (prog->instances.nr < 0 || !prog->instances.fds) { if (prog->preprocessor) { pr_warning("Internal error: can't load program '%s'\n", @@ -1394,10 +1550,9 @@ bpf_program__load(struct bpf_program *prog, pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", prog->section_name, prog->instances.nr); } - err = load_program(prog->type, prog->expected_attach_type, - prog->name, prog->insns, prog->insns_cnt, + err = load_program(prog, prog->insns, prog->insns_cnt, license, kern_version, &fd, - prog->prog_ifindex); + func_info_cnt); if (!err) prog->instances.fds[0] = fd; goto out; @@ -1425,11 +1580,10 @@ bpf_program__load(struct bpf_program *prog, continue; } - err = load_program(prog->type, prog->expected_attach_type, - prog->name, result.new_insn_ptr, + err = load_program(prog, result.new_insn_ptr, result.new_insn_cnt, license, kern_version, &fd, - prog->prog_ifindex); + func_info_cnt); if (err) { pr_warning("Loading the %dth instance of program '%s' failed\n", @@ -1495,12 +1649,12 @@ static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) case BPF_PROG_TYPE_LIRC_MODE2: case BPF_PROG_TYPE_SK_REUSEPORT: case BPF_PROG_TYPE_FLOW_DISSECTOR: - return false; case BPF_PROG_TYPE_UNSPEC: - case BPF_PROG_TYPE_KPROBE: case BPF_PROG_TYPE_TRACEPOINT: - case BPF_PROG_TYPE_PERF_EVENT: case BPF_PROG_TYPE_RAW_TRACEPOINT: + case BPF_PROG_TYPE_PERF_EVENT: + return false; + case BPF_PROG_TYPE_KPROBE: default: return true; } @@ -1627,6 +1781,7 @@ int bpf_object__load(struct bpf_object *obj) obj->loaded = true; + CHECK_ERR(bpf_object__probe_caps(obj), err, out); CHECK_ERR(bpf_object__create_maps(obj), err, out); CHECK_ERR(bpf_object__relocate(obj), err, out); CHECK_ERR(bpf_object__load_progs(obj), err, out); @@ -1699,6 +1854,34 @@ int bpf_program__pin_instance(struct bpf_program *prog, const char *path, return 0; } +int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, + int instance) +{ + int err; + + err = check_path(path); + if (err) + return err; + + if (prog == NULL) { + pr_warning("invalid program pointer\n"); + return -EINVAL; + } + + if (instance < 0 || instance >= prog->instances.nr) { + pr_warning("invalid prog instance %d of prog %s (max %d)\n", + instance, prog->section_name, prog->instances.nr); + return -EINVAL; + } + + err = unlink(path); + if (err != 0) + return -errno; + pr_debug("unpinned program '%s'\n", path); + + return 0; +} + static int make_dir(const char *path) { char *cp, errmsg[STRERR_BUFSIZE]; @@ -1733,6 +1916,11 @@ int bpf_program__pin(struct bpf_program *prog, const char *path) return -EINVAL; } + if (prog->instances.nr == 1) { + /* don't create subdirs when pinning single instance */ + return bpf_program__pin_instance(prog, path, 0); + } + err = make_dir(path); if (err) return err; @@ -1742,16 +1930,83 @@ int bpf_program__pin(struct bpf_program *prog, const char *path) int len; len = snprintf(buf, PATH_MAX, "%s/%d", path, i); + if (len < 0) { + err = -EINVAL; + goto err_unpin; + } else if (len >= PATH_MAX) { + err = -ENAMETOOLONG; + goto err_unpin; + } + + err = bpf_program__pin_instance(prog, buf, i); + if (err) + goto err_unpin; + } + + return 0; + +err_unpin: + for (i = i - 1; i >= 0; i--) { + char buf[PATH_MAX]; + int len; + + len = snprintf(buf, PATH_MAX, "%s/%d", path, i); + if (len < 0) + continue; + else if (len >= PATH_MAX) + continue; + + bpf_program__unpin_instance(prog, buf, i); + } + + rmdir(path); + + return err; +} + +int bpf_program__unpin(struct bpf_program *prog, const char *path) +{ + int i, err; + + err = check_path(path); + if (err) + return err; + + if (prog == NULL) { + pr_warning("invalid program pointer\n"); + return -EINVAL; + } + + if (prog->instances.nr <= 0) { + pr_warning("no instances of prog %s to pin\n", + prog->section_name); + return -EINVAL; + } + + if (prog->instances.nr == 1) { + /* don't create subdirs when pinning single instance */ + return bpf_program__unpin_instance(prog, path, 0); + } + + for (i = 0; i < prog->instances.nr; i++) { + char buf[PATH_MAX]; + int len; + + len = snprintf(buf, PATH_MAX, "%s/%d", path, i); if (len < 0) return -EINVAL; else if (len >= PATH_MAX) return -ENAMETOOLONG; - err = bpf_program__pin_instance(prog, buf, i); + err = bpf_program__unpin_instance(prog, buf, i); if (err) return err; } + err = rmdir(path); + if (err) + return -errno; + return 0; } @@ -1776,12 +2031,33 @@ int bpf_map__pin(struct bpf_map *map, const char *path) } pr_debug("pinned map '%s'\n", path); + return 0; } -int bpf_object__pin(struct bpf_object *obj, const char *path) +int bpf_map__unpin(struct bpf_map *map, const char *path) +{ + int err; + + err = check_path(path); + if (err) + return err; + + if (map == NULL) { + pr_warning("invalid map pointer\n"); + return -EINVAL; + } + + err = unlink(path); + if (err != 0) + return -errno; + pr_debug("unpinned map '%s'\n", path); + + return 0; +} + +int bpf_object__pin_maps(struct bpf_object *obj, const char *path) { - struct bpf_program *prog; struct bpf_map *map; int err; @@ -1803,28 +2079,142 @@ int bpf_object__pin(struct bpf_object *obj, const char *path) len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); + if (len < 0) { + err = -EINVAL; + goto err_unpin_maps; + } else if (len >= PATH_MAX) { + err = -ENAMETOOLONG; + goto err_unpin_maps; + } + + err = bpf_map__pin(map, buf); + if (err) + goto err_unpin_maps; + } + + return 0; + +err_unpin_maps: + while ((map = bpf_map__prev(map, obj))) { + char buf[PATH_MAX]; + int len; + + len = snprintf(buf, PATH_MAX, "%s/%s", path, + bpf_map__name(map)); + if (len < 0) + continue; + else if (len >= PATH_MAX) + continue; + + bpf_map__unpin(map, buf); + } + + return err; +} + +int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) +{ + struct bpf_map *map; + int err; + + if (!obj) + return -ENOENT; + + bpf_map__for_each(map, obj) { + char buf[PATH_MAX]; + int len; + + len = snprintf(buf, PATH_MAX, "%s/%s", path, + bpf_map__name(map)); if (len < 0) return -EINVAL; else if (len >= PATH_MAX) return -ENAMETOOLONG; - err = bpf_map__pin(map, buf); + err = bpf_map__unpin(map, buf); if (err) return err; } + return 0; +} + +int bpf_object__pin_programs(struct bpf_object *obj, const char *path) +{ + struct bpf_program *prog; + int err; + + if (!obj) + return -ENOENT; + + if (!obj->loaded) { + pr_warning("object not yet loaded; load it first\n"); + return -ENOENT; + } + + err = make_dir(path); + if (err) + return err; + + bpf_object__for_each_program(prog, obj) { + char buf[PATH_MAX]; + int len; + + len = snprintf(buf, PATH_MAX, "%s/%s", path, + prog->pin_name); + if (len < 0) { + err = -EINVAL; + goto err_unpin_programs; + } else if (len >= PATH_MAX) { + err = -ENAMETOOLONG; + goto err_unpin_programs; + } + + err = bpf_program__pin(prog, buf); + if (err) + goto err_unpin_programs; + } + + return 0; + +err_unpin_programs: + while ((prog = bpf_program__prev(prog, obj))) { + char buf[PATH_MAX]; + int len; + + len = snprintf(buf, PATH_MAX, "%s/%s", path, + prog->pin_name); + if (len < 0) + continue; + else if (len >= PATH_MAX) + continue; + + bpf_program__unpin(prog, buf); + } + + return err; +} + +int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) +{ + struct bpf_program *prog; + int err; + + if (!obj) + return -ENOENT; + bpf_object__for_each_program(prog, obj) { char buf[PATH_MAX]; int len; len = snprintf(buf, PATH_MAX, "%s/%s", path, - prog->section_name); + prog->pin_name); if (len < 0) return -EINVAL; else if (len >= PATH_MAX) return -ENAMETOOLONG; - err = bpf_program__pin(prog, buf); + err = bpf_program__unpin(prog, buf); if (err) return err; } @@ -1832,6 +2222,23 @@ int bpf_object__pin(struct bpf_object *obj, const char *path) return 0; } +int bpf_object__pin(struct bpf_object *obj, const char *path) +{ + int err; + + err = bpf_object__pin_maps(obj, path); + if (err) + return err; + + err = bpf_object__pin_programs(obj, path); + if (err) { + bpf_object__unpin_maps(obj, path); + return err; + } + + return 0; +} + void bpf_object__close(struct bpf_object *obj) { size_t i; @@ -1845,6 +2252,7 @@ void bpf_object__close(struct bpf_object *obj) bpf_object__elf_finish(obj); bpf_object__unload(obj); btf__free(obj->btf); + btf_ext__free(obj->btf_ext); for (i = 0; i < obj->nr_maps; i++) { zfree(&obj->maps[i].name); @@ -1918,23 +2326,26 @@ void *bpf_object__priv(struct bpf_object *obj) } static struct bpf_program * -__bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) +__bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) { - size_t idx; + size_t nr_programs = obj->nr_programs; + ssize_t idx; - if (!obj->programs) + if (!nr_programs) return NULL; - /* First handler */ - if (prev == NULL) - return &obj->programs[0]; - if (prev->obj != obj) { + if (!p) + /* Iter from the beginning */ + return forward ? &obj->programs[0] : + &obj->programs[nr_programs - 1]; + + if (p->obj != obj) { pr_warning("error: program handler doesn't match object\n"); return NULL; } - idx = (prev - obj->programs) + 1; - if (idx >= obj->nr_programs) + idx = (p - obj->programs) + (forward ? 1 : -1); + if (idx >= obj->nr_programs || idx < 0) return NULL; return &obj->programs[idx]; } @@ -1945,7 +2356,19 @@ bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) struct bpf_program *prog = prev; do { - prog = __bpf_program__next(prog, obj); + prog = __bpf_program__iter(prog, obj, true); + } while (prog && bpf_program__is_function_storage(prog, obj)); + + return prog; +} + +struct bpf_program * +bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) +{ + struct bpf_program *prog = next; + + do { + prog = __bpf_program__iter(prog, obj, false); } while (prog && bpf_program__is_function_storage(prog, obj)); return prog; @@ -2272,10 +2695,24 @@ void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) map->map_ifindex = ifindex; } -struct bpf_map * -bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) +int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) { - size_t idx; + if (!bpf_map_type__is_map_in_map(map->def.type)) { + pr_warning("error: unsupported map type\n"); + return -EINVAL; + } + if (map->inner_map_fd != -1) { + pr_warning("error: inner_map_fd already specified\n"); + return -EINVAL; + } + map->inner_map_fd = fd; + return 0; +} + +static struct bpf_map * +__bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) +{ + ssize_t idx; struct bpf_map *s, *e; if (!obj || !obj->maps) @@ -2284,22 +2721,40 @@ bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) s = obj->maps; e = obj->maps + obj->nr_maps; - if (prev == NULL) - return s; - - if ((prev < s) || (prev >= e)) { + if ((m < s) || (m >= e)) { pr_warning("error in %s: map handler doesn't belong to object\n", __func__); return NULL; } - idx = (prev - obj->maps) + 1; - if (idx >= obj->nr_maps) + idx = (m - obj->maps) + i; + if (idx >= obj->nr_maps || idx < 0) return NULL; return &obj->maps[idx]; } struct bpf_map * +bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) +{ + if (prev == NULL) + return obj->maps; + + return __bpf_map__iter(prev, obj, 1); +} + +struct bpf_map * +bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) +{ + if (next == NULL) { + if (!obj->nr_maps) + return NULL; + return obj->maps + obj->nr_maps - 1; + } + + return __bpf_map__iter(next, obj, -1); +} + +struct bpf_map * bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) { struct bpf_map *pos; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 1f3468dad8b2..f30c3d07bb7d 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -16,6 +16,10 @@ #include <sys/types.h> // for size_t #include <linux/bpf.h> +#ifdef __cplusplus +extern "C" { +#endif + #ifndef LIBBPF_API #define LIBBPF_API __attribute__((visibility("default"))) #endif @@ -71,6 +75,13 @@ struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, LIBBPF_API struct bpf_object *bpf_object__open_buffer(void *obj_buf, size_t obj_buf_sz, const char *name); +LIBBPF_API int bpf_object__pin_maps(struct bpf_object *obj, const char *path); +LIBBPF_API int bpf_object__unpin_maps(struct bpf_object *obj, + const char *path); +LIBBPF_API int bpf_object__pin_programs(struct bpf_object *obj, + const char *path); +LIBBPF_API int bpf_object__unpin_programs(struct bpf_object *obj, + const char *path); LIBBPF_API int bpf_object__pin(struct bpf_object *object, const char *path); LIBBPF_API void bpf_object__close(struct bpf_object *object); @@ -112,6 +123,9 @@ LIBBPF_API struct bpf_program *bpf_program__next(struct bpf_program *prog, (pos) != NULL; \ (pos) = bpf_program__next((pos), (obj))) +LIBBPF_API struct bpf_program *bpf_program__prev(struct bpf_program *prog, + struct bpf_object *obj); + typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, void *); @@ -131,7 +145,11 @@ LIBBPF_API int bpf_program__fd(struct bpf_program *prog); LIBBPF_API int bpf_program__pin_instance(struct bpf_program *prog, const char *path, int instance); +LIBBPF_API int bpf_program__unpin_instance(struct bpf_program *prog, + const char *path, + int instance); LIBBPF_API int bpf_program__pin(struct bpf_program *prog, const char *path); +LIBBPF_API int bpf_program__unpin(struct bpf_program *prog, const char *path); LIBBPF_API void bpf_program__unload(struct bpf_program *prog); struct bpf_insn; @@ -260,6 +278,9 @@ bpf_map__next(struct bpf_map *map, struct bpf_object *obj); (pos) != NULL; \ (pos) = bpf_map__next((pos), (obj))) +LIBBPF_API struct bpf_map * +bpf_map__prev(struct bpf_map *map, struct bpf_object *obj); + LIBBPF_API int bpf_map__fd(struct bpf_map *map); LIBBPF_API const struct bpf_map_def *bpf_map__def(struct bpf_map *map); LIBBPF_API const char *bpf_map__name(struct bpf_map *map); @@ -274,6 +295,9 @@ LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd); LIBBPF_API bool bpf_map__is_offload_neutral(struct bpf_map *map); LIBBPF_API void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex); LIBBPF_API int bpf_map__pin(struct bpf_map *map, const char *path); +LIBBPF_API int bpf_map__unpin(struct bpf_map *map, const char *path); + +LIBBPF_API int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd); LIBBPF_API long libbpf_get_error(const void *ptr); @@ -317,4 +341,9 @@ int libbpf_nl_get_qdisc(int sock, unsigned int nl_pid, int ifindex, libbpf_dump_nlmsg_t dump_qdisc_nlmsg, void *cookie); int libbpf_nl_get_filter(int sock, unsigned int nl_pid, int ifindex, int handle, libbpf_dump_nlmsg_t dump_filter_nlmsg, void *cookie); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + #endif /* __LIBBPF_LIBBPF_H */ diff --git a/tools/lib/bpf/test_libbpf.cpp b/tools/lib/bpf/test_libbpf.cpp new file mode 100644 index 000000000000..abf3fc25c9fa --- /dev/null +++ b/tools/lib/bpf/test_libbpf.cpp @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#include "libbpf.h" +#include "bpf.h" +#include "btf.h" + +/* do nothing, just make sure we can link successfully */ + +int main(int argc, char *argv[]) +{ + /* libbpf.h */ + libbpf_set_print(NULL, NULL, NULL); + + /* bpf.h */ + bpf_prog_get_fd_by_id(0); + + /* btf.h */ + btf__new(NULL, 0, NULL); +} diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 4a54236475ae..73aa6d8f4a2f 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -24,12 +24,13 @@ TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test test_align test_verifier_log test_dev_cgroup test_tcpbpf_user \ test_sock test_btf test_sockmap test_lirc_mode2_user get_cgroup_id_user \ test_socket_cookie test_cgroup_storage test_select_reuseport test_section_names \ - test_netcnt + test_netcnt test_tcpnotify_user TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test_obj_id.o \ test_pkt_md_access.o test_xdp_redirect.o test_xdp_meta.o sockmap_parse_prog.o \ sockmap_verdict_prog.o dev_cgroup.o sample_ret0.o test_tracepoint.o \ test_l4lb_noinline.o test_xdp_noinline.o test_stacktrace_map.o \ + test_tcpnotify_kern.o \ sample_map_ret0.o test_tcpbpf_kern.o test_stacktrace_build_id.o \ sockmap_tcp_msg_prog.o connect4_prog.o connect6_prog.o test_adjust_tail.o \ test_btf_haskv.o test_btf_nokv.o test_sockmap_kern.o test_tunnel_kern.o \ @@ -38,7 +39,7 @@ TEST_GEN_FILES = test_pkt_access.o test_xdp.o test_l4lb.o test_tcp_estats.o test get_cgroup_id_kern.o socket_cookie_prog.o test_select_reuseport_kern.o \ test_skb_cgroup_id_kern.o bpf_flow.o netcnt_prog.o \ test_sk_lookup_kern.o test_xdp_vlan.o test_queue_map.o test_stack_map.o \ - xdp_dummy.o + xdp_dummy.o test_map_in_map.o # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ @@ -75,6 +76,7 @@ $(OUTPUT)/test_sock_addr: cgroup_helpers.c $(OUTPUT)/test_socket_cookie: cgroup_helpers.c $(OUTPUT)/test_sockmap: cgroup_helpers.c $(OUTPUT)/test_tcpbpf_user: cgroup_helpers.c +$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c $(OUTPUT)/test_progs: trace_helpers.c $(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c $(OUTPUT)/test_cgroup_storage: cgroup_helpers.c @@ -125,7 +127,14 @@ $(OUTPUT)/test_stack_map.o: test_queue_stack_map.h BTF_LLC_PROBE := $(shell $(LLC) -march=bpf -mattr=help 2>&1 | grep dwarfris) BTF_PAHOLE_PROBE := $(shell $(BTF_PAHOLE) --help 2>&1 | grep BTF) BTF_OBJCOPY_PROBE := $(shell $(LLVM_OBJCOPY) --help 2>&1 | grep -i 'usage.*llvm') +BTF_LLVM_PROBE := $(shell echo "int main() { return 0; }" | \ + $(CLANG) -target bpf -O2 -g -c -x c - -o ./llvm_btf_verify.o; \ + readelf -S ./llvm_btf_verify.o | grep BTF; \ + /bin/rm -f ./llvm_btf_verify.o) +ifneq ($(BTF_LLVM_PROBE),) + CLANG_FLAGS += -g +else ifneq ($(BTF_LLC_PROBE),) ifneq ($(BTF_PAHOLE_PROBE),) ifneq ($(BTF_OBJCOPY_PROBE),) @@ -135,6 +144,17 @@ ifneq ($(BTF_OBJCOPY_PROBE),) endif endif endif +endif + +# Have one program compiled without "-target bpf" to test whether libbpf loads +# it successfully +$(OUTPUT)/test_xdp.o: test_xdp.c + $(CLANG) $(CLANG_FLAGS) \ + -O2 -emit-llvm -c $< -o - | \ + $(LLC) -march=bpf -mcpu=$(CPU) $(LLC_FLAGS) -filetype=obj -o $@ +ifeq ($(DWARF2BTF),y) + $(BTF_PAHOLE) -J $@ +endif $(OUTPUT)/%.o: %.c $(CLANG) $(CLANG_FLAGS) \ diff --git a/tools/testing/selftests/bpf/bpf_flow.c b/tools/testing/selftests/bpf/bpf_flow.c index 107350a7821d..b9798f558ca7 100644 --- a/tools/testing/selftests/bpf/bpf_flow.c +++ b/tools/testing/selftests/bpf/bpf_flow.c @@ -116,7 +116,7 @@ static __always_inline int parse_eth_proto(struct __sk_buff *skb, __be16 proto) return BPF_DROP; } -SEC("dissect") +SEC("flow_dissector") int _dissect(struct __sk_buff *skb) { if (!skb->vlan_present) diff --git a/tools/testing/selftests/bpf/connect4_prog.c b/tools/testing/selftests/bpf/connect4_prog.c index 5a88a681d2ab..b8395f3c43e9 100644 --- a/tools/testing/selftests/bpf/connect4_prog.c +++ b/tools/testing/selftests/bpf/connect4_prog.c @@ -21,23 +21,48 @@ int _version SEC("version") = 1; SEC("cgroup/connect4") int connect_v4_prog(struct bpf_sock_addr *ctx) { + struct bpf_sock_tuple tuple = {}; struct sockaddr_in sa; + struct bpf_sock *sk; + + /* Verify that new destination is available. */ + memset(&tuple.ipv4.saddr, 0, sizeof(tuple.ipv4.saddr)); + memset(&tuple.ipv4.sport, 0, sizeof(tuple.ipv4.sport)); + + tuple.ipv4.daddr = bpf_htonl(DST_REWRITE_IP4); + tuple.ipv4.dport = bpf_htons(DST_REWRITE_PORT4); + + if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) + return 0; + else if (ctx->type == SOCK_STREAM) + sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple.ipv4), 0, 0); + else + sk = bpf_sk_lookup_udp(ctx, &tuple, sizeof(tuple.ipv4), 0, 0); + + if (!sk) + return 0; + + if (sk->src_ip4 != tuple.ipv4.daddr || + sk->src_port != DST_REWRITE_PORT4) { + bpf_sk_release(sk); + return 0; + } + + bpf_sk_release(sk); /* Rewrite destination. */ ctx->user_ip4 = bpf_htonl(DST_REWRITE_IP4); ctx->user_port = bpf_htons(DST_REWRITE_PORT4); - if (ctx->type == SOCK_DGRAM || ctx->type == SOCK_STREAM) { - ///* Rewrite source. */ - memset(&sa, 0, sizeof(sa)); + /* Rewrite source. */ + memset(&sa, 0, sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_port = bpf_htons(0); - sa.sin_addr.s_addr = bpf_htonl(SRC_REWRITE_IP4); + sa.sin_family = AF_INET; + sa.sin_port = bpf_htons(0); + sa.sin_addr.s_addr = bpf_htonl(SRC_REWRITE_IP4); - if (bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)) != 0) - return 0; - } + if (bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)) != 0) + return 0; return 1; } diff --git a/tools/testing/selftests/bpf/connect6_prog.c b/tools/testing/selftests/bpf/connect6_prog.c index 8ea3f7d12dee..25f5dc7b7aa0 100644 --- a/tools/testing/selftests/bpf/connect6_prog.c +++ b/tools/testing/selftests/bpf/connect6_prog.c @@ -29,7 +29,41 @@ int _version SEC("version") = 1; SEC("cgroup/connect6") int connect_v6_prog(struct bpf_sock_addr *ctx) { + struct bpf_sock_tuple tuple = {}; struct sockaddr_in6 sa; + struct bpf_sock *sk; + + /* Verify that new destination is available. */ + memset(&tuple.ipv6.saddr, 0, sizeof(tuple.ipv6.saddr)); + memset(&tuple.ipv6.sport, 0, sizeof(tuple.ipv6.sport)); + + tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); + tuple.ipv6.daddr[1] = bpf_htonl(DST_REWRITE_IP6_1); + tuple.ipv6.daddr[2] = bpf_htonl(DST_REWRITE_IP6_2); + tuple.ipv6.daddr[3] = bpf_htonl(DST_REWRITE_IP6_3); + + tuple.ipv6.dport = bpf_htons(DST_REWRITE_PORT6); + + if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) + return 0; + else if (ctx->type == SOCK_STREAM) + sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple.ipv6), 0, 0); + else + sk = bpf_sk_lookup_udp(ctx, &tuple, sizeof(tuple.ipv6), 0, 0); + + if (!sk) + return 0; + + if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || + sk->src_ip6[1] != tuple.ipv6.daddr[1] || + sk->src_ip6[2] != tuple.ipv6.daddr[2] || + sk->src_ip6[3] != tuple.ipv6.daddr[3] || + sk->src_port != DST_REWRITE_PORT6) { + bpf_sk_release(sk); + return 0; + } + + bpf_sk_release(sk); /* Rewrite destination. */ ctx->user_ip6[0] = bpf_htonl(DST_REWRITE_IP6_0); @@ -39,21 +73,19 @@ int connect_v6_prog(struct bpf_sock_addr *ctx) ctx->user_port = bpf_htons(DST_REWRITE_PORT6); - if (ctx->type == SOCK_DGRAM || ctx->type == SOCK_STREAM) { - /* Rewrite source. */ - memset(&sa, 0, sizeof(sa)); + /* Rewrite source. */ + memset(&sa, 0, sizeof(sa)); - sa.sin6_family = AF_INET6; - sa.sin6_port = bpf_htons(0); + sa.sin6_family = AF_INET6; + sa.sin6_port = bpf_htons(0); - sa.sin6_addr.s6_addr32[0] = bpf_htonl(SRC_REWRITE_IP6_0); - sa.sin6_addr.s6_addr32[1] = bpf_htonl(SRC_REWRITE_IP6_1); - sa.sin6_addr.s6_addr32[2] = bpf_htonl(SRC_REWRITE_IP6_2); - sa.sin6_addr.s6_addr32[3] = bpf_htonl(SRC_REWRITE_IP6_3); + sa.sin6_addr.s6_addr32[0] = bpf_htonl(SRC_REWRITE_IP6_0); + sa.sin6_addr.s6_addr32[1] = bpf_htonl(SRC_REWRITE_IP6_1); + sa.sin6_addr.s6_addr32[2] = bpf_htonl(SRC_REWRITE_IP6_2); + sa.sin6_addr.s6_addr32[3] = bpf_htonl(SRC_REWRITE_IP6_3); - if (bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)) != 0) - return 0; - } + if (bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)) != 0) + return 0; return 1; } diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c index f42b3396d622..bcbda7037840 100644 --- a/tools/testing/selftests/bpf/test_btf.c +++ b/tools/testing/selftests/bpf/test_btf.c @@ -5,6 +5,7 @@ #include <linux/btf.h> #include <linux/err.h> #include <linux/kernel.h> +#include <linux/filter.h> #include <bpf/bpf.h> #include <sys/resource.h> #include <libelf.h> @@ -22,9 +23,13 @@ #include "bpf_rlimit.h" #include "bpf_util.h" +#define MAX_INSNS 512 +#define MAX_SUBPROGS 16 + static uint32_t pass_cnt; static uint32_t error_cnt; static uint32_t skip_cnt; +static bool jit_enabled; #define CHECK(condition, format...) ({ \ int __ret = !!(condition); \ @@ -60,6 +65,24 @@ static int __base_pr(const char *format, ...) return err; } +static bool is_jit_enabled(void) +{ + const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable"; + bool enabled = false; + int sysctl_fd; + + sysctl_fd = open(jit_sysctl, 0, O_RDONLY); + if (sysctl_fd != -1) { + char tmpc; + + if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1) + enabled = (tmpc != '0'); + close(sysctl_fd); + } + + return enabled; +} + #define BTF_INFO_ENC(kind, root, vlen) \ ((!!(root) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) @@ -85,8 +108,20 @@ static int __base_pr(const char *format, ...) #define BTF_TYPEDEF_ENC(name, type) \ BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0), type) -#define BTF_PTR_ENC(name, type) \ - BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), type) +#define BTF_PTR_ENC(type) \ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), type) + +#define BTF_CONST_ENC(type) \ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), type) + +#define BTF_FUNC_PROTO_ENC(ret_type, nargs) \ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, nargs), ret_type) + +#define BTF_FUNC_PROTO_ARG_ENC(name, type) \ + (name), (type) + +#define BTF_FUNC_ENC(name, func_proto) \ + BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), func_proto) #define BTF_END_RAW 0xdeadbeef #define NAME_TBD 0xdeadb33f @@ -103,6 +138,7 @@ static struct args { bool get_info_test; bool pprint_test; bool always_log; + bool func_type_test; } args; static char btf_log_buf[BTF_LOG_BUF_SIZE]; @@ -1374,6 +1410,464 @@ static struct btf_raw_test raw_tests[] = { .map_create_err = true, }, +{ + .descr = "func proto (int (*)(int, unsigned int))", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* int (*)(int, unsigned int) */ + BTF_FUNC_PROTO_ENC(1, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(0, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + BTF_END_RAW, + }, + .str_sec = "", + .str_sec_size = sizeof(""), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func proto (vararg)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int, unsigned int, ...) */ + BTF_FUNC_PROTO_ENC(0, 3), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(0, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + BTF_FUNC_PROTO_ARG_ENC(0, 0), + BTF_END_RAW, + }, + .str_sec = "", + .str_sec_size = sizeof(""), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func proto (vararg with name)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int b, ... c) */ + BTF_FUNC_PROTO_ENC(0, 3), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 0), + BTF_END_RAW, + }, + .str_sec = "\0a\0b\0c", + .str_sec_size = sizeof("\0a\0b\0c"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid arg#3", +}, + +{ + .descr = "func proto (arg after vararg)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, ..., unsigned int b) */ + BTF_FUNC_PROTO_ENC(0, 3), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 0), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_END_RAW, + }, + .str_sec = "\0a\0b", + .str_sec_size = sizeof("\0a\0b"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid arg#2", +}, + +{ + .descr = "func proto (CONST=>TYPEDEF=>PTR=>FUNC_PROTO)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* typedef void (*func_ptr)(int, unsigned int) */ + BTF_TYPEDEF_ENC(NAME_TBD, 5), /* [3] */ + /* const func_ptr */ + BTF_CONST_ENC(3), /* [4] */ + BTF_PTR_ENC(6), /* [5] */ + BTF_FUNC_PROTO_ENC(0, 2), /* [6] */ + BTF_FUNC_PROTO_ARG_ENC(0, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + BTF_END_RAW, + }, + .str_sec = "\0func_ptr", + .str_sec_size = sizeof("\0func_ptr"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func proto (CONST=>TYPEDEF=>FUNC_PROTO)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + BTF_CONST_ENC(4), /* [3] */ + BTF_TYPEDEF_ENC(NAME_TBD, 5), /* [4] */ + BTF_FUNC_PROTO_ENC(0, 2), /* [5] */ + BTF_FUNC_PROTO_ARG_ENC(0, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + BTF_END_RAW, + }, + .str_sec = "\0func_typedef", + .str_sec_size = sizeof("\0func_typedef"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid type_id", +}, + +{ + .descr = "func proto (btf_resolve(arg))", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* void (*)(const void *) */ + BTF_FUNC_PROTO_ENC(0, 1), /* [2] */ + BTF_FUNC_PROTO_ARG_ENC(0, 3), + BTF_CONST_ENC(4), /* [3] */ + BTF_PTR_ENC(0), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "", + .str_sec_size = sizeof(""), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func proto (Not all arg has name)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int, unsigned int b) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(0, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_END_RAW, + }, + .str_sec = "\0b", + .str_sec_size = sizeof("\0b"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func proto (Bad arg name_off)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int <bad_name_off>) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0xffffffff, 2), + BTF_END_RAW, + }, + .str_sec = "\0a", + .str_sec_size = sizeof("\0a"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid arg#2", +}, + +{ + .descr = "func proto (Bad arg name)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int !!!) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_END_RAW, + }, + .str_sec = "\0a\0!!!", + .str_sec_size = sizeof("\0a\0!!!"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid arg#2", +}, + +{ + .descr = "func proto (Invalid return type)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* <bad_ret_type> (*)(int, unsigned int) */ + BTF_FUNC_PROTO_ENC(100, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(0, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + BTF_END_RAW, + }, + .str_sec = "", + .str_sec_size = sizeof(""), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid return type", +}, + +{ + .descr = "func proto (with func name)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void func_proto(int, unsigned int) */ + BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 2), 0), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(0, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + BTF_END_RAW, + }, + .str_sec = "\0func_proto", + .str_sec_size = sizeof("\0func_proto"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid name", +}, + +{ + .descr = "func proto (const void arg)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(const void) */ + BTF_FUNC_PROTO_ENC(0, 1), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(0, 4), + BTF_CONST_ENC(0), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "", + .str_sec_size = sizeof(""), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_proto_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid arg#1", +}, + +{ + .descr = "func (void func(int a, unsigned int b))", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int b) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + /* void func(int a, unsigned int b) */ + BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0b\0func", + .str_sec_size = sizeof("\0a\0b\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, +}, + +{ + .descr = "func (No func name)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int b) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + /* void <no_name>(int a, unsigned int b) */ + BTF_FUNC_ENC(0, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0b", + .str_sec_size = sizeof("\0a\0b"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid name", +}, + +{ + .descr = "func (Invalid func name)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int b) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + /* void !!!(int a, unsigned int b) */ + BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0b\0!!!", + .str_sec_size = sizeof("\0a\0b\0!!!"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid name", +}, + +{ + .descr = "func (Some arg has no name)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(0, 2), + /* void func(int a, unsigned int) */ + BTF_FUNC_ENC(NAME_TBD, 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0func", + .str_sec_size = sizeof("\0a\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid arg#2", +}, + +{ + .descr = "func (Non zero vlen)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(0, 0, 0, 32, 4), /* [2] */ + /* void (*)(int a, unsigned int b) */ + BTF_FUNC_PROTO_ENC(0, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + /* void func(int a, unsigned int b) */ + BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 2), 3), /* [4] */ + BTF_END_RAW, + }, + .str_sec = "\0a\0b\0func", + .str_sec_size = sizeof("\0a\0b\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "vlen != 0", +}, + +{ + .descr = "func (Not referring to FUNC_PROTO)", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_FUNC_ENC(NAME_TBD, 1), /* [2] */ + BTF_END_RAW, + }, + .str_sec = "\0func", + .str_sec_size = sizeof("\0func"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "func_type_check_btf", + .key_size = sizeof(int), + .value_size = sizeof(int), + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 4, + .btf_load_err = true, + .err_str = "Invalid type_id", +}, + }; /* struct btf_raw_test raw_tests[] */ static const char *get_next_str(const char *start, const char *end) @@ -1940,13 +2434,13 @@ static struct btf_file_test file_tests[] = { }, }; -static int file_has_btf_elf(const char *fn) +static int file_has_btf_elf(const char *fn, bool *has_btf_ext) { Elf_Scn *scn = NULL; GElf_Ehdr ehdr; + int ret = 0; int elf_fd; Elf *elf; - int ret; if (CHECK(elf_version(EV_CURRENT) == EV_NONE, "elf_version(EV_CURRENT) == EV_NONE")) @@ -1978,14 +2472,12 @@ static int file_has_btf_elf(const char *fn) } sh_name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name); - if (!strcmp(sh_name, BTF_ELF_SEC)) { + if (!strcmp(sh_name, BTF_ELF_SEC)) ret = 1; - goto done; - } + if (!strcmp(sh_name, BTF_EXT_ELF_SEC)) + *has_btf_ext = true; } - ret = 0; - done: close(elf_fd); elf_end(elf); @@ -1995,15 +2487,24 @@ done: static int do_test_file(unsigned int test_num) { const struct btf_file_test *test = &file_tests[test_num - 1]; + const char *expected_fnames[] = {"_dummy_tracepoint", + "test_long_fname_1", + "test_long_fname_2"}; + struct bpf_prog_info info = {}; struct bpf_object *obj = NULL; + struct bpf_func_info *finfo; struct bpf_program *prog; + __u32 info_len, rec_size; + bool has_btf_ext = false; + struct btf *btf = NULL; + void *func_info = NULL; struct bpf_map *map; - int err; + int i, err, prog_fd; fprintf(stderr, "BTF libbpf test[%u] (%s): ", test_num, test->file); - err = file_has_btf_elf(test->file); + err = file_has_btf_elf(test->file, &has_btf_ext); if (err == -1) return err; @@ -2031,6 +2532,7 @@ static int do_test_file(unsigned int test_num) err = bpf_object__load(obj); if (CHECK(err < 0, "bpf_object__load: %d", err)) goto done; + prog_fd = bpf_program__fd(prog); map = bpf_object__find_map_by_name(obj, "btf_map"); if (CHECK(!map, "btf_map not found")) { @@ -2045,9 +2547,100 @@ static int do_test_file(unsigned int test_num) test->btf_kv_notfound)) goto done; + if (!jit_enabled || !has_btf_ext) + goto skip_jit; + + /* get necessary program info */ + info_len = sizeof(struct bpf_prog_info); + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + + if (CHECK(err == -1, "invalid get info (1st) errno:%d", errno)) { + fprintf(stderr, "%s\n", btf_log_buf); + err = -1; + goto done; + } + if (CHECK(info.func_info_cnt != 3, + "incorrect info.func_info_cnt (1st) %d", + info.func_info_cnt)) { + err = -1; + goto done; + } + rec_size = info.func_info_rec_size; + if (CHECK(rec_size < 4, + "incorrect info.func_info_rec_size (1st) %d\n", rec_size)) { + err = -1; + goto done; + } + + func_info = malloc(info.func_info_cnt * rec_size); + if (CHECK(!func_info, "out of memory")) { + err = -1; + goto done; + } + + /* reset info to only retrieve func_info related data */ + memset(&info, 0, sizeof(info)); + info.func_info_cnt = 3; + info.func_info_rec_size = rec_size; + info.func_info = ptr_to_u64(func_info); + + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + + if (CHECK(err == -1, "invalid get info (2nd) errno:%d", errno)) { + fprintf(stderr, "%s\n", btf_log_buf); + err = -1; + goto done; + } + if (CHECK(info.func_info_cnt != 3, + "incorrect info.func_info_cnt (2nd) %d", + info.func_info_cnt)) { + err = -1; + goto done; + } + if (CHECK(info.func_info_rec_size != rec_size, + "incorrect info.func_info_rec_size (2nd) %d", + info.func_info_rec_size)) { + err = -1; + goto done; + } + + err = btf_get_from_id(info.btf_id, &btf); + if (CHECK(err, "cannot get btf from kernel, err: %d", err)) + goto done; + + /* check three functions */ + finfo = func_info; + for (i = 0; i < 3; i++) { + const struct btf_type *t; + const char *fname; + + t = btf__type_by_id(btf, finfo->type_id); + if (CHECK(!t, "btf__type_by_id failure: id %u", + finfo->type_id)) { + err = -1; + goto done; + } + + fname = btf__name_by_offset(btf, t->name_off); + err = strcmp(fname, expected_fnames[i]); + /* for the second and third functions in .text section, + * the compiler may order them either way. + */ + if (i && err) + err = strcmp(fname, expected_fnames[3 - i]); + if (CHECK(err, "incorrect fname %s", fname ? : "")) { + err = -1; + goto done; + } + + finfo = (void *)finfo + rec_size; + } + +skip_jit: fprintf(stderr, "OK"); done: + free(func_info); bpf_object__close(obj); return err; } @@ -2477,16 +3070,310 @@ static int test_pprint(void) return err; } +static struct btf_func_type_test { + const char *descr; + const char *str_sec; + __u32 raw_types[MAX_NR_RAW_TYPES]; + __u32 str_sec_size; + struct bpf_insn insns[MAX_INSNS]; + __u32 prog_type; + __u32 func_info[MAX_SUBPROGS][2]; + __u32 func_info_rec_size; + __u32 func_info_cnt; + bool expected_prog_load_failure; +} func_type_test[] = { +{ + .descr = "func_type (main func + one sub)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 32, 4), /* [2] */ + BTF_FUNC_PROTO_ENC(1, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ENC(1, 2), /* [4] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_ENC(NAME_TBD, 3), /* [5] */ + BTF_FUNC_ENC(NAME_TBD, 4), /* [6] */ + BTF_END_RAW, + }, + .str_sec = "\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB", + .str_sec_size = sizeof("\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB"), + .insns = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_IMM(BPF_REG_0, 2), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info = { {0, 5}, {3, 6} }, + .func_info_rec_size = 8, + .func_info_cnt = 2, +}, + +{ + .descr = "func_type (Incorrect func_info_rec_size)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 32, 4), /* [2] */ + BTF_FUNC_PROTO_ENC(1, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ENC(1, 2), /* [4] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_ENC(NAME_TBD, 3), /* [5] */ + BTF_FUNC_ENC(NAME_TBD, 4), /* [6] */ + BTF_END_RAW, + }, + .str_sec = "\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB", + .str_sec_size = sizeof("\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB"), + .insns = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_IMM(BPF_REG_0, 2), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info = { {0, 5}, {3, 6} }, + .func_info_rec_size = 4, + .func_info_cnt = 2, + .expected_prog_load_failure = true, +}, + +{ + .descr = "func_type (Incorrect func_info_cnt)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 32, 4), /* [2] */ + BTF_FUNC_PROTO_ENC(1, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ENC(1, 2), /* [4] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_ENC(NAME_TBD, 3), /* [5] */ + BTF_FUNC_ENC(NAME_TBD, 4), /* [6] */ + BTF_END_RAW, + }, + .str_sec = "\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB", + .str_sec_size = sizeof("\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB"), + .insns = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_IMM(BPF_REG_0, 2), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info = { {0, 5}, {3, 6} }, + .func_info_rec_size = 8, + .func_info_cnt = 1, + .expected_prog_load_failure = true, +}, + +{ + .descr = "func_type (Incorrect bpf_func_info.insn_offset)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 32, 4), /* [2] */ + BTF_FUNC_PROTO_ENC(1, 2), /* [3] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ENC(1, 2), /* [4] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 2), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_ENC(NAME_TBD, 3), /* [5] */ + BTF_FUNC_ENC(NAME_TBD, 4), /* [6] */ + BTF_END_RAW, + }, + .str_sec = "\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB", + .str_sec_size = sizeof("\0int\0unsigned int\0a\0b\0c\0d\0funcA\0funcB"), + .insns = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2), + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_IMM(BPF_REG_0, 2), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info = { {0, 5}, {2, 6} }, + .func_info_rec_size = 8, + .func_info_cnt = 2, + .expected_prog_load_failure = true, +}, + +}; + +static size_t probe_prog_length(const struct bpf_insn *fp) +{ + size_t len; + + for (len = MAX_INSNS - 1; len > 0; --len) + if (fp[len].code != 0 || fp[len].imm != 0) + break; + return len + 1; +} + +static int do_test_func_type(int test_num) +{ + const struct btf_func_type_test *test = &func_type_test[test_num]; + unsigned int raw_btf_size, info_len, rec_size; + int i, btf_fd = -1, prog_fd = -1, err = 0; + struct bpf_load_program_attr attr = {}; + void *raw_btf, *func_info = NULL; + struct bpf_prog_info info = {}; + struct bpf_func_info *finfo; + + fprintf(stderr, "%s......", test->descr); + raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types, + test->str_sec, test->str_sec_size, + &raw_btf_size); + + if (!raw_btf) + return -1; + + *btf_log_buf = '\0'; + btf_fd = bpf_load_btf(raw_btf, raw_btf_size, + btf_log_buf, BTF_LOG_BUF_SIZE, + args.always_log); + free(raw_btf); + + if (CHECK(btf_fd == -1, "invalid btf_fd errno:%d", errno)) { + err = -1; + goto done; + } + + if (*btf_log_buf && args.always_log) + fprintf(stderr, "\n%s", btf_log_buf); + + attr.prog_type = test->prog_type; + attr.insns = test->insns; + attr.insns_cnt = probe_prog_length(attr.insns); + attr.license = "GPL"; + attr.prog_btf_fd = btf_fd; + attr.func_info_rec_size = test->func_info_rec_size; + attr.func_info_cnt = test->func_info_cnt; + attr.func_info = test->func_info; + + *btf_log_buf = '\0'; + prog_fd = bpf_load_program_xattr(&attr, btf_log_buf, + BTF_LOG_BUF_SIZE); + if (test->expected_prog_load_failure && prog_fd == -1) { + err = 0; + goto done; + } + if (CHECK(prog_fd == -1, "invalid prog_id errno:%d", errno)) { + fprintf(stderr, "%s\n", btf_log_buf); + err = -1; + goto done; + } + if (!jit_enabled) { + skip_cnt++; + fprintf(stderr, "SKIPPED, please enable sysctl bpf_jit_enable\n"); + err = 0; + goto done; + } + + /* get necessary lens */ + info_len = sizeof(struct bpf_prog_info); + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + if (CHECK(err == -1, "invalid get info (1st) errno:%d", errno)) { + fprintf(stderr, "%s\n", btf_log_buf); + err = -1; + goto done; + } + if (CHECK(info.func_info_cnt != 2, + "incorrect info.func_info_cnt (1st) %d\n", + info.func_info_cnt)) { + err = -1; + goto done; + } + rec_size = info.func_info_rec_size; + if (CHECK(rec_size < 4, + "incorrect info.func_info_rec_size (1st) %d\n", rec_size)) { + err = -1; + goto done; + } + + func_info = malloc(info.func_info_cnt * rec_size); + if (CHECK(!func_info, "out of memory")) { + err = -1; + goto done; + } + + /* reset info to only retrieve func_info related data */ + memset(&info, 0, sizeof(info)); + info.func_info_cnt = 2; + info.func_info_rec_size = rec_size; + info.func_info = ptr_to_u64(func_info); + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + if (CHECK(err == -1, "invalid get info (2nd) errno:%d", errno)) { + fprintf(stderr, "%s\n", btf_log_buf); + err = -1; + goto done; + } + if (CHECK(info.func_info_cnt != 2, + "incorrect info.func_info_cnt (2nd) %d\n", + info.func_info_cnt)) { + err = -1; + goto done; + } + if (CHECK(info.func_info_rec_size != rec_size, + "incorrect info.func_info_rec_size (2nd) %d\n", + info.func_info_rec_size)) { + err = -1; + goto done; + } + + finfo = func_info; + for (i = 0; i < 2; i++) { + if (CHECK(finfo->type_id != test->func_info[i][1], + "incorrect func_type %u expected %u", + finfo->type_id, test->func_info[i][1])) { + err = -1; + goto done; + } + finfo = (void *)finfo + rec_size; + } + +done: + if (*btf_log_buf && (err || args.always_log)) + fprintf(stderr, "\n%s", btf_log_buf); + + if (btf_fd != -1) + close(btf_fd); + if (prog_fd != -1) + close(prog_fd); + free(func_info); + return err; +} + +static int test_func_type(void) +{ + unsigned int i; + int err = 0; + + for (i = 0; i < ARRAY_SIZE(func_type_test); i++) + err |= count_result(do_test_func_type(i)); + + return err; +} + static void usage(const char *cmd) { - fprintf(stderr, "Usage: %s [-l] [[-r test_num (1 - %zu)] | [-g test_num (1 - %zu)] | [-f test_num (1 - %zu)] | [-p]]\n", + fprintf(stderr, "Usage: %s [-l] [[-r test_num (1 - %zu)] |" + " [-g test_num (1 - %zu)] |" + " [-f test_num (1 - %zu)] | [-p] | [-k] ]\n", cmd, ARRAY_SIZE(raw_tests), ARRAY_SIZE(get_info_tests), ARRAY_SIZE(file_tests)); } static int parse_args(int argc, char **argv) { - const char *optstr = "lpf:r:g:"; + const char *optstr = "lpkf:r:g:"; int opt; while ((opt = getopt(argc, argv, optstr)) != -1) { @@ -2509,6 +3396,9 @@ static int parse_args(int argc, char **argv) case 'p': args.pprint_test = true; break; + case 'k': + args.func_type_test = true; + break; case 'h': usage(argv[0]); exit(0); @@ -2562,6 +3452,8 @@ int main(int argc, char **argv) if (args.always_log) libbpf_set_print(__base_pr, __base_pr, __base_pr); + jit_enabled = is_jit_enabled(); + if (args.raw_test) err |= test_raw(); @@ -2574,8 +3466,11 @@ int main(int argc, char **argv) if (args.pprint_test) err |= test_pprint(); + if (args.func_type_test) + err |= test_func_type(); + if (args.raw_test || args.get_info_test || args.file_test || - args.pprint_test) + args.pprint_test || args.func_type_test) goto done; err |= test_raw(); diff --git a/tools/testing/selftests/bpf/test_btf_haskv.c b/tools/testing/selftests/bpf/test_btf_haskv.c index b21b876f475d..e5c79fe0ffdb 100644 --- a/tools/testing/selftests/bpf/test_btf_haskv.c +++ b/tools/testing/selftests/bpf/test_btf_haskv.c @@ -24,8 +24,8 @@ struct dummy_tracepoint_args { struct sock *sock; }; -SEC("dummy_tracepoint") -int _dummy_tracepoint(struct dummy_tracepoint_args *arg) +__attribute__((noinline)) +static int test_long_fname_2(struct dummy_tracepoint_args *arg) { struct ipv_counts *counts; int key = 0; @@ -42,4 +42,16 @@ int _dummy_tracepoint(struct dummy_tracepoint_args *arg) return 0; } +__attribute__((noinline)) +static int test_long_fname_1(struct dummy_tracepoint_args *arg) +{ + return test_long_fname_2(arg); +} + +SEC("dummy_tracepoint") +int _dummy_tracepoint(struct dummy_tracepoint_args *arg) +{ + return test_long_fname_1(arg); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_btf_nokv.c b/tools/testing/selftests/bpf/test_btf_nokv.c index 0ed8e088eebf..434188c37774 100644 --- a/tools/testing/selftests/bpf/test_btf_nokv.c +++ b/tools/testing/selftests/bpf/test_btf_nokv.c @@ -22,8 +22,8 @@ struct dummy_tracepoint_args { struct sock *sock; }; -SEC("dummy_tracepoint") -int _dummy_tracepoint(struct dummy_tracepoint_args *arg) +__attribute__((noinline)) +static int test_long_fname_2(struct dummy_tracepoint_args *arg) { struct ipv_counts *counts; int key = 0; @@ -40,4 +40,16 @@ int _dummy_tracepoint(struct dummy_tracepoint_args *arg) return 0; } +__attribute__((noinline)) +static int test_long_fname_1(struct dummy_tracepoint_args *arg) +{ + return test_long_fname_2(arg); +} + +SEC("dummy_tracepoint") +int _dummy_tracepoint(struct dummy_tracepoint_args *arg) +{ + return test_long_fname_1(arg); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_flow_dissector.sh b/tools/testing/selftests/bpf/test_flow_dissector.sh index c0fb073b5eab..d23d4da66b83 100755 --- a/tools/testing/selftests/bpf/test_flow_dissector.sh +++ b/tools/testing/selftests/bpf/test_flow_dissector.sh @@ -59,7 +59,7 @@ else fi # Attach BPF program -./flow_dissector_load -p bpf_flow.o -s dissect +./flow_dissector_load -p bpf_flow.o -s flow_dissector # Setup tc qdisc add dev lo ingress diff --git a/tools/testing/selftests/bpf/test_libbpf.sh b/tools/testing/selftests/bpf/test_libbpf.sh index 156d89f1edcc..2989b2e2d856 100755 --- a/tools/testing/selftests/bpf/test_libbpf.sh +++ b/tools/testing/selftests/bpf/test_libbpf.sh @@ -33,17 +33,11 @@ trap exit_handler 0 2 3 6 9 libbpf_open_file test_l4lb.o -# TODO: fix libbpf to load noinline functions -# [warning] libbpf: incorrect bpf_call opcode -#libbpf_open_file test_l4lb_noinline.o +# Load a program with BPF-to-BPF calls +libbpf_open_file test_l4lb_noinline.o -# TODO: fix test_xdp_meta.c to load with libbpf -# [warning] libbpf: test_xdp_meta.o doesn't provide kernel version -#libbpf_open_file test_xdp_meta.o - -# TODO: fix libbpf to handle .eh_frame -# [warning] libbpf: relocation failed: no section(10) -#libbpf_open_file ../../../../samples/bpf/tracex3_kern.o +# Load a program compiled without the "-target bpf" flag +libbpf_open_file test_xdp.o # Success exit 0 diff --git a/tools/testing/selftests/bpf/test_map_in_map.c b/tools/testing/selftests/bpf/test_map_in_map.c new file mode 100644 index 000000000000..ce923e67e08e --- /dev/null +++ b/tools/testing/selftests/bpf/test_map_in_map.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2018 Facebook */ +#include <stddef.h> +#include <linux/bpf.h> +#include <linux/types.h> +#include "bpf_helpers.h" + +struct bpf_map_def SEC("maps") mim_array = { + .type = BPF_MAP_TYPE_ARRAY_OF_MAPS, + .key_size = sizeof(int), + /* must be sizeof(__u32) for map in map */ + .value_size = sizeof(__u32), + .max_entries = 1, + .map_flags = 0, +}; + +struct bpf_map_def SEC("maps") mim_hash = { + .type = BPF_MAP_TYPE_HASH_OF_MAPS, + .key_size = sizeof(int), + /* must be sizeof(__u32) for map in map */ + .value_size = sizeof(__u32), + .max_entries = 1, + .map_flags = 0, +}; + +SEC("xdp_mimtest") +int xdp_mimtest0(struct xdp_md *ctx) +{ + int value = 123; + int key = 0; + void *map; + + map = bpf_map_lookup_elem(&mim_array, &key); + if (!map) + return XDP_DROP; + + bpf_map_update_elem(map, &key, &value, 0); + + map = bpf_map_lookup_elem(&mim_hash, &key); + if (!map) + return XDP_DROP; + + bpf_map_update_elem(map, &key, &value, 0); + + return XDP_PASS; +} + +int _version SEC("version") = 1; +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index 4db2116e52be..9c79ee017df3 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -258,24 +258,36 @@ static void test_hashmap_percpu(int task, void *data) close(fd); } -static void test_hashmap_walk(int task, void *data) +static int helper_fill_hashmap(int max_entries) { - int fd, i, max_entries = 1000; - long long key, value, next_key; - bool next_key_valid = true; + int i, fd, ret; + long long key, value; fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value), max_entries, map_flags); - if (fd < 0) { - printf("Failed to create hashmap '%s'!\n", strerror(errno)); - exit(1); - } + CHECK(fd < 0, + "failed to create hashmap", + "err: %s, flags: 0x%x\n", strerror(errno), map_flags); for (i = 0; i < max_entries; i++) { key = i; value = key; - assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0); + ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST); + CHECK(ret != 0, + "can't update hashmap", + "err: %s\n", strerror(ret)); } + return fd; +} + +static void test_hashmap_walk(int task, void *data) +{ + int fd, i, max_entries = 1000; + long long key, value, next_key; + bool next_key_valid = true; + + fd = helper_fill_hashmap(max_entries); + for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key, &next_key) == 0; i++) { key = next_key; @@ -306,6 +318,39 @@ static void test_hashmap_walk(int task, void *data) close(fd); } +static void test_hashmap_zero_seed(void) +{ + int i, first, second, old_flags; + long long key, next_first, next_second; + + old_flags = map_flags; + map_flags |= BPF_F_ZERO_SEED; + + first = helper_fill_hashmap(3); + second = helper_fill_hashmap(3); + + for (i = 0; ; i++) { + void *key_ptr = !i ? NULL : &key; + + if (bpf_map_get_next_key(first, key_ptr, &next_first) != 0) + break; + + CHECK(bpf_map_get_next_key(second, key_ptr, &next_second) != 0, + "next_key for second map must succeed", + "key_ptr: %p", key_ptr); + CHECK(next_first != next_second, + "keys must match", + "i: %d first: %lld second: %lld\n", i, + next_first, next_second); + + key = next_first; + } + + map_flags = old_flags; + close(first); + close(second); +} + static void test_arraymap(int task, void *data) { int key, next_key, fd; @@ -1080,6 +1125,94 @@ out_sockmap: exit(1); } +#define MAPINMAP_PROG "./test_map_in_map.o" +static void test_map_in_map(void) +{ + struct bpf_program *prog; + struct bpf_object *obj; + struct bpf_map *map; + int mim_fd, fd, err; + int pos = 0; + + obj = bpf_object__open(MAPINMAP_PROG); + + fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(int), sizeof(int), + 2, 0); + if (fd < 0) { + printf("Failed to create hashmap '%s'!\n", strerror(errno)); + exit(1); + } + + map = bpf_object__find_map_by_name(obj, "mim_array"); + if (IS_ERR(map)) { + printf("Failed to load array of maps from test prog\n"); + goto out_map_in_map; + } + err = bpf_map__set_inner_map_fd(map, fd); + if (err) { + printf("Failed to set inner_map_fd for array of maps\n"); + goto out_map_in_map; + } + + map = bpf_object__find_map_by_name(obj, "mim_hash"); + if (IS_ERR(map)) { + printf("Failed to load hash of maps from test prog\n"); + goto out_map_in_map; + } + err = bpf_map__set_inner_map_fd(map, fd); + if (err) { + printf("Failed to set inner_map_fd for hash of maps\n"); + goto out_map_in_map; + } + + bpf_object__for_each_program(prog, obj) { + bpf_program__set_xdp(prog); + } + bpf_object__load(obj); + + map = bpf_object__find_map_by_name(obj, "mim_array"); + if (IS_ERR(map)) { + printf("Failed to load array of maps from test prog\n"); + goto out_map_in_map; + } + mim_fd = bpf_map__fd(map); + if (mim_fd < 0) { + printf("Failed to get descriptor for array of maps\n"); + goto out_map_in_map; + } + + err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); + if (err) { + printf("Failed to update array of maps\n"); + goto out_map_in_map; + } + + map = bpf_object__find_map_by_name(obj, "mim_hash"); + if (IS_ERR(map)) { + printf("Failed to load hash of maps from test prog\n"); + goto out_map_in_map; + } + mim_fd = bpf_map__fd(map); + if (mim_fd < 0) { + printf("Failed to get descriptor for hash of maps\n"); + goto out_map_in_map; + } + + err = bpf_map_update_elem(mim_fd, &pos, &fd, 0); + if (err) { + printf("Failed to update hash of maps\n"); + goto out_map_in_map; + } + + close(fd); + bpf_object__close(obj); + return; + +out_map_in_map: + close(fd); + exit(1); +} + #define MAP_SIZE (32 * 1024) static void test_map_large(void) @@ -1534,6 +1667,7 @@ static void run_all_tests(void) test_hashmap(0, NULL); test_hashmap_percpu(0, NULL); test_hashmap_walk(0, NULL); + test_hashmap_zero_seed(); test_arraymap(0, NULL); test_arraymap_percpu(0, NULL); @@ -1554,6 +1688,8 @@ static void run_all_tests(void) test_queuemap(0, NULL); test_stackmap(0, NULL); + + test_map_in_map(); } int main(void) diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 2d3c04f45530..c1e688f61061 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -1703,7 +1703,7 @@ static void test_reference_tracking() const char *file = "./test_sk_lookup_kern.o"; struct bpf_object *obj; struct bpf_program *prog; - __u32 duration; + __u32 duration = 0; int err = 0; obj = bpf_object__open(file); diff --git a/tools/testing/selftests/bpf/test_sock_addr.c b/tools/testing/selftests/bpf/test_sock_addr.c index aeeb76a54d63..73b7493d4120 100644 --- a/tools/testing/selftests/bpf/test_sock_addr.c +++ b/tools/testing/selftests/bpf/test_sock_addr.c @@ -574,24 +574,44 @@ static int bind4_prog_load(const struct sock_addr_test *test) /* if (sk.family == AF_INET && */ BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6, offsetof(struct bpf_sock_addr, family)), - BPF_JMP_IMM(BPF_JNE, BPF_REG_7, AF_INET, 16), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, AF_INET, 24), /* (sk.type == SOCK_DGRAM || sk.type == SOCK_STREAM) && */ BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6, offsetof(struct bpf_sock_addr, type)), BPF_JMP_IMM(BPF_JNE, BPF_REG_7, SOCK_DGRAM, 1), BPF_JMP_A(1), - BPF_JMP_IMM(BPF_JNE, BPF_REG_7, SOCK_STREAM, 12), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, SOCK_STREAM, 20), /* 1st_byte_of_user_ip4 == expected && */ BPF_LDX_MEM(BPF_B, BPF_REG_7, BPF_REG_6, offsetof(struct bpf_sock_addr, user_ip4)), - BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr8[0], 10), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr8[0], 18), + + /* 2nd_byte_of_user_ip4 == expected && */ + BPF_LDX_MEM(BPF_B, BPF_REG_7, BPF_REG_6, + offsetof(struct bpf_sock_addr, user_ip4) + 1), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr8[1], 16), + + /* 3rd_byte_of_user_ip4 == expected && */ + BPF_LDX_MEM(BPF_B, BPF_REG_7, BPF_REG_6, + offsetof(struct bpf_sock_addr, user_ip4) + 2), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr8[2], 14), + + /* 4th_byte_of_user_ip4 == expected && */ + BPF_LDX_MEM(BPF_B, BPF_REG_7, BPF_REG_6, + offsetof(struct bpf_sock_addr, user_ip4) + 3), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr8[3], 12), /* 1st_half_of_user_ip4 == expected && */ BPF_LDX_MEM(BPF_H, BPF_REG_7, BPF_REG_6, offsetof(struct bpf_sock_addr, user_ip4)), - BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr16[0], 8), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr16[0], 10), + + /* 2nd_half_of_user_ip4 == expected && */ + BPF_LDX_MEM(BPF_H, BPF_REG_7, BPF_REG_6, + offsetof(struct bpf_sock_addr, user_ip4) + 2), + BPF_JMP_IMM(BPF_JNE, BPF_REG_7, ip4.u4_addr16[1], 8), /* whole_user_ip4 == expected) { */ BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6, diff --git a/tools/testing/selftests/bpf/test_tcpnotify.h b/tools/testing/selftests/bpf/test_tcpnotify.h new file mode 100644 index 000000000000..8b6cea030bfc --- /dev/null +++ b/tools/testing/selftests/bpf/test_tcpnotify.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifndef _TEST_TCPBPF_H +#define _TEST_TCPBPF_H + +struct tcpnotify_globals { + __u32 total_retrans; + __u32 ncalls; +}; + +struct tcp_notifier { + __u8 type; + __u8 subtype; + __u8 source; + __u8 hash; +}; + +#define TESTPORT 12877 +#endif diff --git a/tools/testing/selftests/bpf/test_tcpnotify_kern.c b/tools/testing/selftests/bpf/test_tcpnotify_kern.c new file mode 100644 index 000000000000..edbca203ce2d --- /dev/null +++ b/tools/testing/selftests/bpf/test_tcpnotify_kern.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stddef.h> +#include <string.h> +#include <linux/bpf.h> +#include <linux/if_ether.h> +#include <linux/if_packet.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/types.h> +#include <linux/socket.h> +#include <linux/tcp.h> +#include <netinet/in.h> +#include "bpf_helpers.h" +#include "bpf_endian.h" +#include "test_tcpnotify.h" + +struct bpf_map_def SEC("maps") global_map = { + .type = BPF_MAP_TYPE_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct tcpnotify_globals), + .max_entries = 4, +}; + +struct bpf_map_def SEC("maps") perf_event_map = { + .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY, + .key_size = sizeof(int), + .value_size = sizeof(__u32), + .max_entries = 2, +}; + +int _version SEC("version") = 1; + +SEC("sockops") +int bpf_testcb(struct bpf_sock_ops *skops) +{ + int rv = -1; + int op; + + op = (int) skops->op; + + if (bpf_ntohl(skops->remote_port) != TESTPORT) { + skops->reply = -1; + return 0; + } + + switch (op) { + case BPF_SOCK_OPS_TIMEOUT_INIT: + case BPF_SOCK_OPS_RWND_INIT: + case BPF_SOCK_OPS_NEEDS_ECN: + case BPF_SOCK_OPS_BASE_RTT: + case BPF_SOCK_OPS_RTO_CB: + rv = 1; + break; + + case BPF_SOCK_OPS_TCP_CONNECT_CB: + case BPF_SOCK_OPS_TCP_LISTEN_CB: + case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB: + case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: + bpf_sock_ops_cb_flags_set(skops, (BPF_SOCK_OPS_RETRANS_CB_FLAG| + BPF_SOCK_OPS_RTO_CB_FLAG)); + rv = 1; + break; + case BPF_SOCK_OPS_RETRANS_CB: { + __u32 key = 0; + struct tcpnotify_globals g, *gp; + struct tcp_notifier msg = { + .type = 0xde, + .subtype = 0xad, + .source = 0xbe, + .hash = 0xef, + }; + + rv = 1; + + /* Update results */ + gp = bpf_map_lookup_elem(&global_map, &key); + if (!gp) + break; + g = *gp; + g.total_retrans = skops->total_retrans; + g.ncalls++; + bpf_map_update_elem(&global_map, &key, &g, + BPF_ANY); + bpf_perf_event_output(skops, &perf_event_map, + BPF_F_CURRENT_CPU, + &msg, sizeof(msg)); + } + break; + default: + rv = -1; + } + skops->reply = rv; + return 1; +} +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_tcpnotify_user.c b/tools/testing/selftests/bpf/test_tcpnotify_user.c new file mode 100644 index 000000000000..ff3c4522aed6 --- /dev/null +++ b/tools/testing/selftests/bpf/test_tcpnotify_user.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include <pthread.h> +#include <inttypes.h> +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <asm/types.h> +#include <sys/syscall.h> +#include <errno.h> +#include <string.h> +#include <linux/bpf.h> +#include <sys/socket.h> +#include <bpf/bpf.h> +#include <bpf/libbpf.h> +#include <sys/ioctl.h> +#include <linux/rtnetlink.h> +#include <signal.h> +#include <linux/perf_event.h> + +#include "bpf_rlimit.h" +#include "bpf_util.h" +#include "cgroup_helpers.h" + +#include "test_tcpnotify.h" +#include "trace_helpers.h" + +#define SOCKET_BUFFER_SIZE (getpagesize() < 8192L ? getpagesize() : 8192L) + +pthread_t tid; +int rx_callbacks; + +static int dummyfn(void *data, int size) +{ + struct tcp_notifier *t = data; + + if (t->type != 0xde || t->subtype != 0xad || + t->source != 0xbe || t->hash != 0xef) + return 1; + rx_callbacks++; + return 0; +} + +void tcp_notifier_poller(int fd) +{ + while (1) + perf_event_poller(fd, dummyfn); +} + +static void *poller_thread(void *arg) +{ + int fd = *(int *)arg; + + tcp_notifier_poller(fd); + return arg; +} + +int verify_result(const struct tcpnotify_globals *result) +{ + return (result->ncalls > 0 && result->ncalls == rx_callbacks ? 0 : 1); +} + +static int bpf_find_map(const char *test, struct bpf_object *obj, + const char *name) +{ + struct bpf_map *map; + + map = bpf_object__find_map_by_name(obj, name); + if (!map) { + printf("%s:FAIL:map '%s' not found\n", test, name); + return -1; + } + return bpf_map__fd(map); +} + +static int setup_bpf_perf_event(int mapfd) +{ + struct perf_event_attr attr = { + .sample_type = PERF_SAMPLE_RAW, + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_BPF_OUTPUT, + }; + int key = 0; + int pmu_fd; + + pmu_fd = syscall(__NR_perf_event_open, &attr, -1, 0, -1, 0); + if (pmu_fd < 0) + return pmu_fd; + bpf_map_update_elem(mapfd, &key, &pmu_fd, BPF_ANY); + + ioctl(pmu_fd, PERF_EVENT_IOC_ENABLE, 0); + return pmu_fd; +} + +int main(int argc, char **argv) +{ + const char *file = "test_tcpnotify_kern.o"; + int prog_fd, map_fd, perf_event_fd; + struct tcpnotify_globals g = {0}; + const char *cg_path = "/foo"; + int error = EXIT_FAILURE; + struct bpf_object *obj; + int cg_fd = -1; + __u32 key = 0; + int rv; + char test_script[80]; + int pmu_fd; + cpu_set_t cpuset; + + CPU_ZERO(&cpuset); + CPU_SET(0, &cpuset); + pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + + if (setup_cgroup_environment()) + goto err; + + cg_fd = create_and_get_cgroup(cg_path); + if (!cg_fd) + goto err; + + if (join_cgroup(cg_path)) + goto err; + + if (bpf_prog_load(file, BPF_PROG_TYPE_SOCK_OPS, &obj, &prog_fd)) { + printf("FAILED: load_bpf_file failed for: %s\n", file); + goto err; + } + + rv = bpf_prog_attach(prog_fd, cg_fd, BPF_CGROUP_SOCK_OPS, 0); + if (rv) { + printf("FAILED: bpf_prog_attach: %d (%s)\n", + error, strerror(errno)); + goto err; + } + + perf_event_fd = bpf_find_map(__func__, obj, "perf_event_map"); + if (perf_event_fd < 0) + goto err; + + map_fd = bpf_find_map(__func__, obj, "global_map"); + if (map_fd < 0) + goto err; + + pmu_fd = setup_bpf_perf_event(perf_event_fd); + if (pmu_fd < 0 || perf_event_mmap(pmu_fd) < 0) + goto err; + + pthread_create(&tid, NULL, poller_thread, (void *)&pmu_fd); + + sprintf(test_script, + "/usr/sbin/iptables -A INPUT -p tcp --dport %d -j DROP", + TESTPORT); + system(test_script); + + sprintf(test_script, + "/usr/bin/nc 127.0.0.1 %d < /etc/passwd > /dev/null 2>&1 ", + TESTPORT); + system(test_script); + + sprintf(test_script, + "/usr/sbin/iptables -D INPUT -p tcp --dport %d -j DROP", + TESTPORT); + system(test_script); + + rv = bpf_map_lookup_elem(map_fd, &key, &g); + if (rv != 0) { + printf("FAILED: bpf_map_lookup_elem returns %d\n", rv); + goto err; + } + + sleep(10); + + if (verify_result(&g)) { + printf("FAILED: Wrong stats Expected %d calls, got %d\n", + g.ncalls, rx_callbacks); + goto err; + } + + printf("PASSED!\n"); + error = 0; +err: + bpf_prog_detach(cg_fd, BPF_CGROUP_SOCK_OPS); + close(cg_fd); + cleanup_cgroup_environment(); + return error; +} diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 6f61df62f690..537a8f91af02 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -2026,29 +2026,27 @@ static struct bpf_test tests[] = { .result = ACCEPT, }, { - "check skb->hash byte load not permitted 1", + "check skb->hash byte load permitted 1", .insns = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, offsetof(struct __sk_buff, hash) + 1), BPF_EXIT_INSN(), }, - .errstr = "invalid bpf_context access", - .result = REJECT, + .result = ACCEPT, }, { - "check skb->hash byte load not permitted 2", + "check skb->hash byte load permitted 2", .insns = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, offsetof(struct __sk_buff, hash) + 2), BPF_EXIT_INSN(), }, - .errstr = "invalid bpf_context access", - .result = REJECT, + .result = ACCEPT, }, { - "check skb->hash byte load not permitted 3", + "check skb->hash byte load permitted 3", .insns = { BPF_MOV64_IMM(BPF_REG_0, 0), #if __BYTE_ORDER == __LITTLE_ENDIAN @@ -2060,8 +2058,7 @@ static struct bpf_test tests[] = { #endif BPF_EXIT_INSN(), }, - .errstr = "invalid bpf_context access", - .result = REJECT, + .result = ACCEPT, }, { "check cb access: byte, wrong type", @@ -2173,7 +2170,7 @@ static struct bpf_test tests[] = { .result = ACCEPT, }, { - "check skb->hash half load not permitted", + "check skb->hash half load permitted 2", .insns = { BPF_MOV64_IMM(BPF_REG_0, 0), #if __BYTE_ORDER == __LITTLE_ENDIAN @@ -2185,6 +2182,37 @@ static struct bpf_test tests[] = { #endif BPF_EXIT_INSN(), }, + .result = ACCEPT, + }, + { + "check skb->hash half load not permitted, unaligned 1", + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 0), +#if __BYTE_ORDER == __LITTLE_ENDIAN + BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, hash) + 1), +#else + BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, hash) + 3), +#endif + BPF_EXIT_INSN(), + }, + .errstr = "invalid bpf_context access", + .result = REJECT, + }, + { + "check skb->hash half load not permitted, unaligned 3", + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 0), +#if __BYTE_ORDER == __LITTLE_ENDIAN + BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, hash) + 3), +#else + BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, hash) + 1), +#endif + BPF_EXIT_INSN(), + }, .errstr = "invalid bpf_context access", .result = REJECT, }, @@ -2418,6 +2446,10 @@ static struct bpf_test tests[] = { offsetof(struct __sk_buff, tc_index)), BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_0, offsetof(struct __sk_buff, cb[3])), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, tstamp)), + BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, + offsetof(struct __sk_buff, tstamp)), BPF_EXIT_INSN(), }, .errstr_unpriv = "", @@ -5270,6 +5302,31 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_CGROUP_SKB, }, { + "write tstamp from CGROUP_SKB", + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0, + offsetof(struct __sk_buff, tstamp)), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .result = ACCEPT, + .result_unpriv = REJECT, + .errstr_unpriv = "invalid bpf_context access off=152 size=8", + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { + "read tstamp from CGROUP_SKB", + .insns = { + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, tstamp)), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .result = ACCEPT, + .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + }, + { "multiple registers share map_lookup_elem result", .insns = { BPF_MOV64_IMM(BPF_REG_1, 10), |