aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/btf.h1
-rw-r--r--include/uapi/linux/bpf.h2
-rw-r--r--kernel/bpf/helpers.c4
-rw-r--r--kernel/bpf/verifier.c5
-rwxr-xr-xscripts/bpf_doc.py53
-rw-r--r--tools/bpf/bpftool/btf.c98
-rw-r--r--tools/include/uapi/linux/bpf.h2
7 files changed, 146 insertions, 19 deletions
diff --git a/include/linux/btf.h b/include/linux/btf.h
index b8a583194c4a..631060e3ad14 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -75,6 +75,7 @@
#define KF_ITER_NEXT (1 << 9) /* kfunc implements BPF iter next method */
#define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
#define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
+#define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */
/*
* Tag marking a kernel function as a kfunc. This is meant to minimize the
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index c6cd7c7aeeee..8ab4d8184b9d 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -1970,6 +1970,8 @@ union bpf_attr {
* program.
* Return
* The SMP id of the processor running the program.
+ * Attributes
+ * __bpf_fastcall
*
* long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
* Description
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 1a43d06eab28..4053f279ed4c 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -3052,8 +3052,8 @@ BTF_ID(func, bpf_cgroup_release_dtor)
#endif
BTF_KFUNCS_START(common_btf_ids)
-BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx)
-BTF_ID_FLAGS(func, bpf_rdonly_cast)
+BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx, KF_FASTCALL)
+BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL)
BTF_ID_FLAGS(func, bpf_rcu_read_lock)
BTF_ID_FLAGS(func, bpf_rcu_read_unlock)
BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL)
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 9a7ed527e47e..7d9b38ffd220 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -16176,10 +16176,7 @@ static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)
/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
{
- if (meta->btf == btf_vmlinux)
- return meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
- meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast];
- return false;
+ return meta->kfunc_flags & KF_FASTCALL;
}
/* LLVM define a bpf_fastcall function attribute.
diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py
index c55878bddfdd..e74a01a85070 100755
--- a/scripts/bpf_doc.py
+++ b/scripts/bpf_doc.py
@@ -37,10 +37,11 @@ class APIElement(object):
@desc: textual description of the symbol
@ret: (optional) description of any associated return value
"""
- def __init__(self, proto='', desc='', ret=''):
+ def __init__(self, proto='', desc='', ret='', attrs=[]):
self.proto = proto
self.desc = desc
self.ret = ret
+ self.attrs = attrs
class Helper(APIElement):
@@ -81,6 +82,11 @@ class Helper(APIElement):
return res
+ATTRS = {
+ '__bpf_fastcall': 'bpf_fastcall'
+}
+
+
class HeaderParser(object):
"""
An object used to parse a file in order to extract the documentation of a
@@ -111,7 +117,8 @@ class HeaderParser(object):
proto = self.parse_proto()
desc = self.parse_desc(proto)
ret = self.parse_ret(proto)
- return Helper(proto=proto, desc=desc, ret=ret)
+ attrs = self.parse_attrs(proto)
+ return Helper(proto=proto, desc=desc, ret=ret, attrs=attrs)
def parse_symbol(self):
p = re.compile(r' \* ?(BPF\w+)$')
@@ -192,6 +199,28 @@ class HeaderParser(object):
raise Exception("No return found for " + proto)
return ret
+ def parse_attrs(self, proto):
+ p = re.compile(r' \* ?(?:\t| {5,8})Attributes$')
+ capture = p.match(self.line)
+ if not capture:
+ return []
+ # Expect a single line with mnemonics for attributes separated by spaces
+ self.line = self.reader.readline()
+ p = re.compile(r' \* ?(?:\t| {5,8})(?:\t| {8})(.*)')
+ capture = p.match(self.line)
+ if not capture:
+ raise Exception("Incomplete 'Attributes' section for " + proto)
+ attrs = capture.group(1).split(' ')
+ for attr in attrs:
+ if attr not in ATTRS:
+ raise Exception("Unexpected attribute '" + attr + "' specified for " + proto)
+ self.line = self.reader.readline()
+ if self.line != ' *\n':
+ raise Exception("Expecting empty line after 'Attributes' section for " + proto)
+ # Prepare a line for next self.parse_* to consume
+ self.line = self.reader.readline()
+ return attrs
+
def seek_to(self, target, help_message, discard_lines = 1):
self.reader.seek(0)
offset = self.reader.read().find(target)
@@ -789,6 +818,21 @@ class PrinterHelpers(Printer):
print('%s;' % fwd)
print('')
+ used_attrs = set()
+ for helper in self.elements:
+ for attr in helper.attrs:
+ used_attrs.add(attr)
+ for attr in sorted(used_attrs):
+ print('#ifndef %s' % attr)
+ print('#if __has_attribute(%s)' % ATTRS[attr])
+ print('#define %s __attribute__((%s))' % (attr, ATTRS[attr]))
+ print('#else')
+ print('#define %s' % attr)
+ print('#endif')
+ print('#endif')
+ if used_attrs:
+ print('')
+
def print_footer(self):
footer = ''
print(footer)
@@ -827,7 +871,10 @@ class PrinterHelpers(Printer):
print(' *{}{}'.format(' \t' if line else '', line))
print(' */')
- print('static %s %s(* const %s)(' % (self.map_type(proto['ret_type']),
+ print('static ', end='')
+ if helper.attrs:
+ print('%s ' % (" ".join(helper.attrs)), end='')
+ print('%s %s(* const %s)(' % (self.map_type(proto['ret_type']),
proto['ret_star'], proto['name']), end='')
comma = ''
for i, a in enumerate(proto['args']):
diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c
index 7d2af1ff3c8d..547c1ccdce98 100644
--- a/tools/bpf/bpftool/btf.c
+++ b/tools/bpf/bpftool/btf.c
@@ -1,11 +1,15 @@
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
/* Copyright (C) 2019 Facebook */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
#include <errno.h>
#include <fcntl.h>
#include <linux/err.h>
#include <stdbool.h>
#include <stdio.h>
+#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <linux/btf.h>
@@ -21,6 +25,7 @@
#include "main.h"
#define KFUNC_DECL_TAG "bpf_kfunc"
+#define FASTCALL_DECL_TAG "bpf_fastcall"
static const char * const btf_kind_str[NR_BTF_KINDS] = {
[BTF_KIND_UNKN] = "UNKNOWN",
@@ -464,19 +469,59 @@ static int dump_btf_raw(const struct btf *btf,
return 0;
}
+struct ptr_array {
+ __u32 cnt;
+ __u32 cap;
+ const void **elems;
+};
+
+static int ptr_array_push(const void *ptr, struct ptr_array *arr)
+{
+ __u32 new_cap;
+ void *tmp;
+
+ if (arr->cnt == arr->cap) {
+ new_cap = (arr->cap ?: 16) * 2;
+ tmp = realloc(arr->elems, sizeof(*arr->elems) * new_cap);
+ if (!tmp)
+ return -ENOMEM;
+ arr->elems = tmp;
+ arr->cap = new_cap;
+ }
+ arr->elems[arr->cnt++] = ptr;
+ return 0;
+}
+
+static void ptr_array_free(struct ptr_array *arr)
+{
+ free(arr->elems);
+}
+
+static int cmp_kfuncs(const void *pa, const void *pb, void *ctx)
+{
+ struct btf *btf = ctx;
+ const struct btf_type *a = *(void **)pa;
+ const struct btf_type *b = *(void **)pb;
+
+ return strcmp(btf__str_by_offset(btf, a->name_off),
+ btf__str_by_offset(btf, b->name_off));
+}
+
static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf)
{
LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts);
- int cnt = btf__type_cnt(btf);
- int i;
+ __u32 cnt = btf__type_cnt(btf), i, j;
+ struct ptr_array fastcalls = {};
+ struct ptr_array kfuncs = {};
+ int err = 0;
printf("\n/* BPF kfuncs */\n");
printf("#ifndef BPF_NO_KFUNC_PROTOTYPES\n");
for (i = 1; i < cnt; i++) {
const struct btf_type *t = btf__type_by_id(btf, i);
+ const struct btf_type *ft;
const char *name;
- int err;
if (!btf_is_decl_tag(t))
continue;
@@ -484,27 +529,53 @@ static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf)
if (btf_decl_tag(t)->component_idx != -1)
continue;
- name = btf__name_by_offset(btf, t->name_off);
- if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)))
+ ft = btf__type_by_id(btf, t->type);
+ if (!btf_is_func(ft))
continue;
- t = btf__type_by_id(btf, t->type);
- if (!btf_is_func(t))
- continue;
+ name = btf__name_by_offset(btf, t->name_off);
+ if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)) == 0) {
+ err = ptr_array_push(ft, &kfuncs);
+ if (err)
+ goto out;
+ }
+
+ if (strncmp(name, FASTCALL_DECL_TAG, sizeof(FASTCALL_DECL_TAG)) == 0) {
+ err = ptr_array_push(ft, &fastcalls);
+ if (err)
+ goto out;
+ }
+ }
+
+ /* Sort kfuncs by name for improved vmlinux.h stability */
+ qsort_r(kfuncs.elems, kfuncs.cnt, sizeof(*kfuncs.elems), cmp_kfuncs, (void *)btf);
+ for (i = 0; i < kfuncs.cnt; i++) {
+ const struct btf_type *t = kfuncs.elems[i];
printf("extern ");
+ /* Assume small amount of fastcall kfuncs */
+ for (j = 0; j < fastcalls.cnt; j++) {
+ if (fastcalls.elems[j] == t) {
+ printf("__bpf_fastcall ");
+ break;
+ }
+ }
+
opts.field_name = btf__name_by_offset(btf, t->name_off);
err = btf_dump__emit_type_decl(d, t->type, &opts);
if (err)
- return err;
+ goto out;
printf(" __weak __ksym;\n");
}
printf("#endif\n\n");
- return 0;
+out:
+ ptr_array_free(&fastcalls);
+ ptr_array_free(&kfuncs);
+ return err;
}
static void __printf(2, 0) btf_dump_printf(void *ctx,
@@ -718,6 +789,13 @@ static int dump_btf_c(const struct btf *btf,
printf("#ifndef __weak\n");
printf("#define __weak __attribute__((weak))\n");
printf("#endif\n\n");
+ printf("#ifndef __bpf_fastcall\n");
+ printf("#if __has_attribute(bpf_fastcall)\n");
+ printf("#define __bpf_fastcall __attribute__((bpf_fastcall))\n");
+ printf("#else\n");
+ printf("#define __bpf_fastcall\n");
+ printf("#endif\n");
+ printf("#endif\n\n");
if (root_type_cnt) {
for (i = 0; i < root_type_cnt; i++) {
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 1fb3cb2636e6..7610883c8191 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -1970,6 +1970,8 @@ union bpf_attr {
* program.
* Return
* The SMP id of the processor running the program.
+ * Attributes
+ * __bpf_fastcall
*
* long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
* Description
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.