diff options
Diffstat (limited to 'arch/riscv')
24 files changed, 573 insertions, 256 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index fa7dc03459e7..73f029eae0cc 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,8 +12,6 @@ config 32BIT config RISCV def_bool y - # even on 32-bit, physical (and DMA) addresses are > 32-bits - select PHYS_ADDR_T_64BIT select OF select OF_EARLY_FLATTREE select OF_IRQ @@ -57,6 +55,7 @@ config RISCV select GENERIC_ARCH_TOPOLOGY if SMP select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_MMIOWB + select ARCH_HAS_DEBUG_VIRTUAL select HAVE_EBPF_JIT if 64BIT select EDAC_SUPPORT select ARCH_HAS_GIGANTIC_PAGE @@ -66,6 +65,7 @@ config RISCV select HAVE_ARCH_MMAP_RND_BITS if MMU select ARCH_HAS_GCOV_PROFILE_ALL select HAVE_COPY_THREAD_TLS + select HAVE_ARCH_KASAN if MMU && 64BIT config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT diff --git a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi index a2e3d54e830c..7db861053483 100644 --- a/arch/riscv/boot/dts/sifive/fu540-c000.dtsi +++ b/arch/riscv/boot/dts/sifive/fu540-c000.dtsi @@ -268,6 +268,19 @@ interrupts = <1 2 3>; reg = <0x0 0x2010000 0x0 0x1000>; }; - + gpio: gpio@10060000 { + compatible = "sifive,fu540-c000-gpio", "sifive,gpio0"; + interrupt-parent = <&plic0>; + interrupts = <7>, <8>, <9>, <10>, <11>, <12>, <13>, + <14>, <15>, <16>, <17>, <18>, <19>, <20>, + <21>, <22>; + reg = <0x0 0x10060000 0x0 0x1000>; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&prci PRCI_CLK_TLCLK>; + status = "disabled"; + }; }; }; diff --git a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts index 88cfcb96bf23..609198cb1163 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts @@ -94,3 +94,7 @@ &pwm1 { status = "okay"; }; + +&gpio { + status = "okay"; +}; diff --git a/arch/riscv/include/asm/Kbuild b/arch/riscv/include/asm/Kbuild index 1efaeddf1e4b..ec0ca8c6ab64 100644 --- a/arch/riscv/include/asm/Kbuild +++ b/arch/riscv/include/asm/Kbuild @@ -7,7 +7,6 @@ generic-y += div64.h generic-y += extable.h generic-y += flat.h generic-y += dma.h -generic-y += dma-contiguous.h generic-y += dma-mapping.h generic-y += emergency-restart.h generic-y += exec.h diff --git a/arch/riscv/include/asm/image.h b/arch/riscv/include/asm/image.h index 7b0f92ba0acc..e0b319af3681 100644 --- a/arch/riscv/include/asm/image.h +++ b/arch/riscv/include/asm/image.h @@ -42,7 +42,7 @@ * @res2: reserved * @magic: Magic number (RISC-V specific; deprecated) * @magic2: Magic number 2 (to match the ARM64 'magic' field pos) - * @res4: reserved (will be used for PE COFF offset) + * @res3: reserved (will be used for PE COFF offset) * * The intention is for this header format to be shared between multiple * architectures to avoid a proliferation of image header formats. @@ -59,7 +59,7 @@ struct riscv_image_header { u64 res2; u64 magic; u32 magic2; - u32 res4; + u32 res3; }; #endif /* __ASSEMBLY__ */ #endif /* _ASM_RISCV_IMAGE_H */ diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h new file mode 100644 index 000000000000..eee6e6588b12 --- /dev/null +++ b/arch/riscv/include/asm/kasan.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2019 Andes Technology Corporation */ + +#ifndef __ASM_KASAN_H +#define __ASM_KASAN_H + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_KASAN + +#include <asm/pgtable.h> + +#define KASAN_SHADOW_SCALE_SHIFT 3 + +#define KASAN_SHADOW_SIZE (UL(1) << (38 - KASAN_SHADOW_SCALE_SHIFT)) +#define KASAN_SHADOW_START 0xffffffc000000000 /* 2^64 - 2^38 */ +#define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) + +#define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << \ + (64 - KASAN_SHADOW_SCALE_SHIFT))) + +void kasan_init(void); +asmlinkage void kasan_early_init(void); + +#endif +#endif +#endif /* __ASM_KASAN_H */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index ac699246ae7e..8ca1930caa44 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -100,8 +100,20 @@ extern unsigned long pfn_base; extern unsigned long max_low_pfn; extern unsigned long min_low_pfn; -#define __pa(x) ((unsigned long)(x) - va_pa_offset) -#define __va(x) ((void *)((unsigned long) (x) + va_pa_offset)) +#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset)) +#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset) + +#ifdef CONFIG_DEBUG_VIRTUAL +extern phys_addr_t __virt_to_phys(unsigned long x); +extern phys_addr_t __phys_addr_symbol(unsigned long x); +#else +#define __virt_to_phys(x) __va_to_pa_nodebug(x) +#define __phys_addr_symbol(x) __va_to_pa_nodebug(x) +#endif /* CONFIG_DEBUG_VIRTUAL */ + +#define __pa_symbol(x) __phys_addr_symbol(RELOC_HIDE((unsigned long)(x), 0)) +#define __pa(x) __virt_to_phys((unsigned long)(x)) +#define __va(x) ((void *)__pa_to_va_nodebug((phys_addr_t)(x))) #define phys_to_pfn(phys) (PFN_DOWN(phys)) #define pfn_to_phys(pfn) (PFN_PHYS(pfn)) diff --git a/arch/riscv/include/asm/perf_event.h b/arch/riscv/include/asm/perf_event.h index aefbfaa6a781..0234048b12bc 100644 --- a/arch/riscv/include/asm/perf_event.h +++ b/arch/riscv/include/asm/perf_event.h @@ -82,4 +82,8 @@ struct riscv_pmu { int irq; }; +#ifdef CONFIG_PERF_EVENTS +#define perf_arch_bpf_user_pt_regs(regs) (struct user_regs_struct *)regs +#endif + #endif /* _ASM_RISCV_PERF_EVENT_H */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 74630989006d..b15f70a1fdfa 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -43,6 +43,13 @@ static inline int pud_bad(pud_t pud) return !pud_present(pud); } +#define pud_leaf pud_leaf +static inline int pud_leaf(pud_t pud) +{ + return pud_present(pud) && + (pud_val(pud) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); +} + static inline void set_pud(pud_t *pudp, pud_t pud) { *pudp = pud; @@ -58,6 +65,11 @@ static inline unsigned long pud_page_vaddr(pud_t pud) return (unsigned long)pfn_to_virt(pud_val(pud) >> _PAGE_PFN_SHIFT); } +static inline struct page *pud_page(pud_t pud) +{ + return pfn_to_page(pud_val(pud) >> _PAGE_PFN_SHIFT); +} + #define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 36ae01761352..e43041519edd 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -94,6 +94,10 @@ extern pgd_t swapper_pg_dir[]; #define VMALLOC_END (PAGE_OFFSET - 1) #define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE) +#define BPF_JIT_REGION_SIZE (SZ_128M) +#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE) +#define BPF_JIT_REGION_END (VMALLOC_END) + /* * Roughly size the vmemmap space to be large enough to fit enough * struct pages to map half the virtual address space. Then @@ -126,6 +130,13 @@ static inline int pmd_bad(pmd_t pmd) return !pmd_present(pmd); } +#define pmd_leaf pmd_leaf +static inline int pmd_leaf(pmd_t pmd) +{ + return pmd_present(pmd) && + (pmd_val(pmd) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); +} + static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; diff --git a/arch/riscv/include/asm/string.h b/arch/riscv/include/asm/string.h index 1b5d44585962..924af13f8555 100644 --- a/arch/riscv/include/asm/string.h +++ b/arch/riscv/include/asm/string.h @@ -11,8 +11,17 @@ #define __HAVE_ARCH_MEMSET extern asmlinkage void *memset(void *, int, size_t); +extern asmlinkage void *__memset(void *, int, size_t); #define __HAVE_ARCH_MEMCPY extern asmlinkage void *memcpy(void *, const void *, size_t); +extern asmlinkage void *__memcpy(void *, const void *, size_t); +/* For those files which don't want to check by kasan. */ +#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__) + +#define memcpy(dst, src, len) __memcpy(dst, src, len) +#define memset(s, c, n) __memset(s, c, n) + +#endif #endif /* _ASM_RISCV_STRING_H */ diff --git a/arch/riscv/include/asm/vmalloc.h b/arch/riscv/include/asm/vmalloc.h new file mode 100644 index 000000000000..ff9abc00d139 --- /dev/null +++ b/arch/riscv/include/asm/vmalloc.h @@ -0,0 +1,4 @@ +#ifndef _ASM_RISCV_VMALLOC_H +#define _ASM_RISCV_VMALLOC_H + +#endif /* _ASM_RISCV_VMALLOC_H */ diff --git a/arch/riscv/include/uapi/asm/bpf_perf_event.h b/arch/riscv/include/uapi/asm/bpf_perf_event.h new file mode 100644 index 000000000000..6cb1c2823288 --- /dev/null +++ b/arch/riscv/include/uapi/asm/bpf_perf_event.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__ASM_BPF_PERF_EVENT_H__ +#define _UAPI__ASM_BPF_PERF_EVENT_H__ + +#include <asm/ptrace.h> + +typedef struct user_regs_struct bpf_user_pt_regs_t; + +#endif /* _UAPI__ASM_BPF_PERF_EVENT_H__ */ diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index e163b7b64c86..bad4d85b5e91 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -155,7 +155,7 @@ _save_context: REG_L x2, PT_SP(sp) .endm -#if !IS_ENABLED(CONFIG_PREEMPT) +#if !IS_ENABLED(CONFIG_PREEMPTION) .set resume_kernel, restore_all #endif @@ -305,7 +305,7 @@ restore_all: sret #endif -#if IS_ENABLED(CONFIG_PREEMPT) +#if IS_ENABLED(CONFIG_PREEMPTION) resume_kernel: REG_L s0, TASK_TI_PREEMPT_COUNT(tp) bnez s0, restore_all diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index a4242be66966..271860fc2c3f 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -121,6 +121,9 @@ clear_bss_done: sw zero, TASK_TI_CPU(tp) la sp, init_thread_union + THREAD_SIZE +#ifdef CONFIG_KASAN + call kasan_early_init +#endif /* Start the kernel */ call parse_dtb tail start_kernel diff --git a/arch/riscv/kernel/riscv_ksyms.c b/arch/riscv/kernel/riscv_ksyms.c index 2a02b7eebee0..450492e1cb4e 100644 --- a/arch/riscv/kernel/riscv_ksyms.c +++ b/arch/riscv/kernel/riscv_ksyms.c @@ -11,3 +11,5 @@ */ EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(__memcpy); diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index 365ff8420bfe..0a6d415b0a5a 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -24,6 +24,7 @@ #include <asm/smp.h> #include <asm/tlbflush.h> #include <asm/thread_info.h> +#include <asm/kasan.h> #include "head.h" @@ -74,12 +75,12 @@ void __init setup_arch(char **cmdline_p) swiotlb_init(1); #endif -#ifdef CONFIG_SMP - setup_smp(); +#ifdef CONFIG_KASAN + kasan_init(); #endif -#ifdef CONFIG_DUMMY_CONSOLE - conswitchp = &dummy_con; +#ifdef CONFIG_SMP + setup_smp(); #endif riscv_fill_hwcap(); diff --git a/arch/riscv/kernel/vmlinux.lds.S b/arch/riscv/kernel/vmlinux.lds.S index 12f42f96d46e..1e0193ded420 100644 --- a/arch/riscv/kernel/vmlinux.lds.S +++ b/arch/riscv/kernel/vmlinux.lds.S @@ -46,6 +46,7 @@ SECTIONS KPROBES_TEXT ENTRY_TEXT IRQENTRY_TEXT + SOFTIRQENTRY_TEXT *(.fixup) _etext = .; } diff --git a/arch/riscv/lib/memcpy.S b/arch/riscv/lib/memcpy.S index b4c477846e91..51ab716253fa 100644 --- a/arch/riscv/lib/memcpy.S +++ b/arch/riscv/lib/memcpy.S @@ -7,7 +7,8 @@ #include <asm/asm.h> /* void *memcpy(void *, const void *, size_t) */ -ENTRY(memcpy) +ENTRY(__memcpy) +WEAK(memcpy) move t6, a0 /* Preserve return value */ /* Defer to byte-oriented copy for small sizes */ @@ -104,4 +105,4 @@ ENTRY(memcpy) bltu a1, a3, 5b 6: ret -END(memcpy) +END(__memcpy) diff --git a/arch/riscv/lib/memset.S b/arch/riscv/lib/memset.S index 5a7386b47175..34c5360c6705 100644 --- a/arch/riscv/lib/memset.S +++ b/arch/riscv/lib/memset.S @@ -8,7 +8,8 @@ #include <asm/asm.h> /* void *memset(void *, int, size_t) */ -ENTRY(memset) +ENTRY(__memset) +WEAK(memset) move t0, a0 /* Preserve return value */ /* Defer to byte-oriented fill for small sizes */ @@ -109,4 +110,4 @@ ENTRY(memset) bltu t0, a3, 5b 6: ret -END(memset) +END(__memset) diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index a1bd95c8047a..50b7af58c566 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -15,3 +15,11 @@ ifeq ($(CONFIG_MMU),y) obj-$(CONFIG_SMP) += tlbflush.o endif obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_KASAN) += kasan_init.o + +ifdef CONFIG_KASAN +KASAN_SANITIZE_kasan_init.o := n +KASAN_SANITIZE_init.o := n +endif + +obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c new file mode 100644 index 000000000000..f0cc86040587 --- /dev/null +++ b/arch/riscv/mm/kasan_init.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2019 Andes Technology Corporation + +#include <linux/pfn.h> +#include <linux/init_task.h> +#include <linux/kasan.h> +#include <linux/kernel.h> +#include <linux/memblock.h> +#include <asm/tlbflush.h> +#include <asm/pgtable.h> +#include <asm/fixmap.h> + +extern pgd_t early_pg_dir[PTRS_PER_PGD]; +asmlinkage void __init kasan_early_init(void) +{ + uintptr_t i; + pgd_t *pgd = early_pg_dir + pgd_index(KASAN_SHADOW_START); + + for (i = 0; i < PTRS_PER_PTE; ++i) + set_pte(kasan_early_shadow_pte + i, + mk_pte(virt_to_page(kasan_early_shadow_page), + PAGE_KERNEL)); + + for (i = 0; i < PTRS_PER_PMD; ++i) + set_pmd(kasan_early_shadow_pmd + i, + pfn_pmd(PFN_DOWN(__pa((uintptr_t)kasan_early_shadow_pte)), + __pgprot(_PAGE_TABLE))); + + for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END; + i += PGDIR_SIZE, ++pgd) + set_pgd(pgd, + pfn_pgd(PFN_DOWN(__pa(((uintptr_t)kasan_early_shadow_pmd))), + __pgprot(_PAGE_TABLE))); + + /* init for swapper_pg_dir */ + pgd = pgd_offset_k(KASAN_SHADOW_START); + + for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END; + i += PGDIR_SIZE, ++pgd) + set_pgd(pgd, + pfn_pgd(PFN_DOWN(__pa(((uintptr_t)kasan_early_shadow_pmd))), + __pgprot(_PAGE_TABLE))); + + flush_tlb_all(); +} + +static void __init populate(void *start, void *end) +{ + unsigned long i; + unsigned long vaddr = (unsigned long)start & PAGE_MASK; + unsigned long vend = PAGE_ALIGN((unsigned long)end); + unsigned long n_pages = (vend - vaddr) / PAGE_SIZE; + unsigned long n_pmds = + (n_pages % PTRS_PER_PTE) ? n_pages / PTRS_PER_PTE + 1 : + n_pages / PTRS_PER_PTE; + pgd_t *pgd = pgd_offset_k(vaddr); + pmd_t *pmd = memblock_alloc(n_pmds * sizeof(pmd_t), PAGE_SIZE); + pte_t *pte = memblock_alloc(n_pages * sizeof(pte_t), PAGE_SIZE); + + for (i = 0; i < n_pages; i++) { + phys_addr_t phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + + set_pte(pte + i, pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); + } + + for (i = 0; i < n_pmds; ++pgd, i += PTRS_PER_PMD) + set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(((uintptr_t)(pmd + i)))), + __pgprot(_PAGE_TABLE))); + + for (i = 0; i < n_pages; ++pmd, i += PTRS_PER_PTE) + set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa((uintptr_t)(pte + i))), + __pgprot(_PAGE_TABLE))); + + flush_tlb_all(); + memset(start, 0, end - start); +} + +void __init kasan_init(void) +{ + struct memblock_region *reg; + unsigned long i; + + kasan_populate_early_shadow((void *)KASAN_SHADOW_START, + (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); + + for_each_memblock(memory, reg) { + void *start = (void *)__va(reg->base); + void *end = (void *)__va(reg->base + reg->size); + + if (start >= end) + break; + + populate(kasan_mem_to_shadow(start), + kasan_mem_to_shadow(end)); + }; + + for (i = 0; i < PTRS_PER_PTE; i++) + set_pte(&kasan_early_shadow_pte[i], + mk_pte(virt_to_page(kasan_early_shadow_page), + __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_ACCESSED))); + + memset(kasan_early_shadow_page, 0, PAGE_SIZE); + init_task.kasan_depth = 0; +} diff --git a/arch/riscv/mm/physaddr.c b/arch/riscv/mm/physaddr.c new file mode 100644 index 000000000000..e8e4dcd39fed --- /dev/null +++ b/arch/riscv/mm/physaddr.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/types.h> +#include <linux/mmdebug.h> +#include <linux/mm.h> +#include <asm/page.h> +#include <asm/sections.h> + +phys_addr_t __virt_to_phys(unsigned long x) +{ + phys_addr_t y = x - PAGE_OFFSET; + + /* + * Boundary checking aginst the kernel linear mapping space. + */ + WARN(y >= KERN_VIRT_SIZE, + "virt_to_phys used for non-linear address: %pK (%pS)\n", + (void *)x, (void *)x); + + return __va_to_pa_nodebug(x); +} +EXPORT_SYMBOL(__virt_to_phys); + +phys_addr_t __phys_addr_symbol(unsigned long x) +{ + unsigned long kernel_start = (unsigned long)PAGE_OFFSET; + unsigned long kernel_end = (unsigned long)_end; + + /* + * Boundary checking aginst the kernel image mapping. + * __pa_symbol should only be used on kernel symbol addresses. + */ + VIRTUAL_BUG_ON(x < kernel_start || x > kernel_end); + + return __va_to_pa_nodebug(x); +} +EXPORT_SYMBOL(__phys_addr_symbol); diff --git a/arch/riscv/net/bpf_jit_comp.c b/arch/riscv/net/bpf_jit_comp.c index 7fbf56aab661..483f4ad7f4dc 100644 --- a/arch/riscv/net/bpf_jit_comp.c +++ b/arch/riscv/net/bpf_jit_comp.c @@ -120,6 +120,11 @@ static bool seen_reg(int reg, struct rv_jit_context *ctx) return false; } +static void mark_fp(struct rv_jit_context *ctx) +{ + __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); +} + static void mark_call(struct rv_jit_context *ctx) { __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); @@ -456,6 +461,11 @@ static u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); } +static u32 rv_auipc(u8 rd, u32 imm31_12) +{ + return rv_u_insn(imm31_12, rd, 0x17); +} + static bool is_12b_int(s64 val) { return -(1 << 11) <= val && val < (1 << 11); @@ -479,27 +489,7 @@ static bool is_32b_int(s64 val) static int is_12b_check(int off, int insn) { if (!is_12b_int(off)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", - insn, (int)off); - return -1; - } - return 0; -} - -static int is_13b_check(int off, int insn) -{ - if (!is_13b_int(off)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", - insn, (int)off); - return -1; - } - return 0; -} - -static int is_21b_check(int off, int insn) -{ - if (!is_21b_int(off)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", + pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", insn, (int)off); return -1; } @@ -545,10 +535,13 @@ static void emit_imm(u8 rd, s64 val, struct rv_jit_context *ctx) emit(rv_addi(rd, rd, lower), ctx); } -static int rv_offset(int bpf_to, int bpf_from, struct rv_jit_context *ctx) +static int rv_offset(int insn, int off, struct rv_jit_context *ctx) { - int from = ctx->offset[bpf_from] - 1, to = ctx->offset[bpf_to]; + int from, to; + off++; /* BPF branch is from PC+1, RV is from PC */ + from = (insn > 0) ? ctx->offset[insn - 1] : 0; + to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0; return (to - from) << 2; } @@ -559,7 +552,7 @@ static int epilogue_offset(struct rv_jit_context *ctx) return (to - from) << 2; } -static void __build_epilogue(u8 reg, struct rv_jit_context *ctx) +static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) { int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; @@ -596,8 +589,114 @@ static void __build_epilogue(u8 reg, struct rv_jit_context *ctx) emit(rv_addi(RV_REG_SP, RV_REG_SP, stack_adjust), ctx); /* Set return value. */ - emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); - emit(rv_jalr(RV_REG_ZERO, reg, 0), ctx); + if (!is_tail_call) + emit(rv_addi(RV_REG_A0, RV_REG_A5, 0), ctx); + emit(rv_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, + is_tail_call ? 4 : 0), /* skip TCC init */ + ctx); +} + +/* return -1 or inverted cond */ +static int invert_bpf_cond(u8 cond) +{ + switch (cond) { + case BPF_JEQ: + return BPF_JNE; + case BPF_JGT: + return BPF_JLE; + case BPF_JLT: + return BPF_JGE; + case BPF_JGE: + return BPF_JLT; + case BPF_JLE: + return BPF_JGT; + case BPF_JNE: + return BPF_JEQ; + case BPF_JSGT: + return BPF_JSLE; + case BPF_JSLT: + return BPF_JSGE; + case BPF_JSGE: + return BPF_JSLT; + case BPF_JSLE: + return BPF_JSGT; + } + return -1; +} + +static void emit_bcc(u8 cond, u8 rd, u8 rs, int rvoff, + struct rv_jit_context *ctx) +{ + switch (cond) { + case BPF_JEQ: + emit(rv_beq(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JGT: + emit(rv_bltu(rs, rd, rvoff >> 1), ctx); + return; + case BPF_JLT: + emit(rv_bltu(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JGE: + emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JLE: + emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); + return; + case BPF_JNE: + emit(rv_bne(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JSGT: + emit(rv_blt(rs, rd, rvoff >> 1), ctx); + return; + case BPF_JSLT: + emit(rv_blt(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JSGE: + emit(rv_bge(rd, rs, rvoff >> 1), ctx); + return; + case BPF_JSLE: + emit(rv_bge(rs, rd, rvoff >> 1), ctx); + } +} + +static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, + struct rv_jit_context *ctx) +{ + s64 upper, lower; + + if (is_13b_int(rvoff)) { + emit_bcc(cond, rd, rs, rvoff, ctx); + return; + } + + /* Adjust for jal */ + rvoff -= 4; + + /* Transform, e.g.: + * bne rd,rs,foo + * to + * beq rd,rs,<.L1> + * (auipc foo) + * jal(r) foo + * .L1 + */ + cond = invert_bpf_cond(cond); + if (is_21b_int(rvoff)) { + emit_bcc(cond, rd, rs, 8, ctx); + emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); + return; + } + + /* 32b No need for an additional rvoff adjustment, since we + * get that from the auipc at PC', where PC = PC' + 4. + */ + upper = (rvoff + (1 << 11)) >> 12; + lower = rvoff & 0xfff; + + emit_bcc(cond, rd, rs, 12, ctx); + emit(rv_auipc(RV_REG_T1, upper), ctx); + emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); } static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) @@ -627,18 +726,14 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) return -1; emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - if (is_13b_check(off, insn)) - return -1; - emit(rv_bgeu(RV_REG_A2, RV_REG_T1, off >> 1), ctx); + emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); /* if (TCC-- < 0) * goto out; */ emit(rv_addi(RV_REG_T1, tcc, -1), ctx); off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - if (is_13b_check(off, insn)) - return -1; - emit(rv_blt(tcc, RV_REG_ZERO, off >> 1), ctx); + emit_branch(BPF_JSLT, tcc, RV_REG_ZERO, off, ctx); /* prog = array->ptrs[index]; * if (!prog) @@ -651,18 +746,15 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) return -1; emit(rv_ld(RV_REG_T2, off, RV_REG_T2), ctx); off = (tc_ninsn - (ctx->ninsns - start_insn)) << 2; - if (is_13b_check(off, insn)) - return -1; - emit(rv_beq(RV_REG_T2, RV_REG_ZERO, off >> 1), ctx); + emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); /* goto *(prog->bpf_func + 4); */ off = offsetof(struct bpf_prog, bpf_func); if (is_12b_check(off, insn)) return -1; emit(rv_ld(RV_REG_T3, off, RV_REG_T2), ctx); - emit(rv_addi(RV_REG_T3, RV_REG_T3, 4), ctx); emit(rv_addi(RV_REG_TCC, RV_REG_T1, 0), ctx); - __build_epilogue(RV_REG_T3, ctx); + __build_epilogue(true, ctx); return 0; } @@ -687,13 +779,6 @@ static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, *rs = bpf_to_rv_reg(insn->src_reg, ctx); } -static int rv_offset_check(int *rvoff, s16 off, int insn, - struct rv_jit_context *ctx) -{ - *rvoff = rv_offset(insn + off, insn, ctx); - return is_13b_check(*rvoff, insn); -} - static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) { emit(rv_addi(RV_REG_T2, *rd, 0), ctx); @@ -726,13 +811,57 @@ static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) *rd = RV_REG_T2; } +static void emit_jump_and_link(u8 rd, s64 rvoff, bool force_jalr, + struct rv_jit_context *ctx) +{ + s64 upper, lower; + + if (rvoff && is_21b_int(rvoff) && !force_jalr) { + emit(rv_jal(rd, rvoff >> 1), ctx); + return; + } + + upper = (rvoff + (1 << 11)) >> 12; + lower = rvoff & 0xfff; + emit(rv_auipc(RV_REG_T1, upper), ctx); + emit(rv_jalr(rd, RV_REG_T1, lower), ctx); +} + +static bool is_signed_bpf_cond(u8 cond) +{ + return cond == BPF_JSGT || cond == BPF_JSLT || + cond == BPF_JSGE || cond == BPF_JSLE; +} + +static int emit_call(bool fixed, u64 addr, struct rv_jit_context *ctx) +{ + s64 off = 0; + u64 ip; + u8 rd; + + if (addr && ctx->insns) { + ip = (u64)(long)(ctx->insns + ctx->ninsns); + off = addr - ip; + if (!is_32b_int(off)) { + pr_err("bpf-jit: target call addr %pK is out of range\n", + (void *)addr); + return -ERANGE; + } + } + + emit_jump_and_link(RV_REG_RA, off, !fixed, ctx); + rd = bpf_to_rv_reg(BPF_REG_0, ctx); + emit(rv_addi(rd, RV_REG_A0, 0), ctx); + return 0; +} + static int emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, bool extra_pass) { bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || BPF_CLASS(insn->code) == BPF_JMP; + int s, e, rvoff, i = insn - ctx->prog->insnsi; struct bpf_prog_aux *aux = ctx->prog->aux; - int rvoff, i = insn - ctx->prog->insnsi; u8 rd = -1, rs = -1, code = insn->code; s16 off = insn->off; s32 imm = insn->imm; @@ -1000,214 +1129,110 @@ out_be: /* JUMP off */ case BPF_JMP | BPF_JA: - rvoff = rv_offset(i + off, i, ctx); - if (!is_21b_int(rvoff)) { - pr_err("bpf-jit: insn=%d offset=%d not supported yet!\n", - i, rvoff); - return -1; - } - - emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); + rvoff = rv_offset(i, off, ctx); + emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); break; /* IF (dst COND src) JUMP off */ case BPF_JMP | BPF_JEQ | BPF_X: case BPF_JMP32 | BPF_JEQ | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_beq(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGT | BPF_X: case BPF_JMP32 | BPF_JGT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bltu(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLT | BPF_X: case BPF_JMP32 | BPF_JLT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bltu(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGE | BPF_X: case BPF_JMP32 | BPF_JGE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bgeu(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLE | BPF_X: case BPF_JMP32 | BPF_JLE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bgeu(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JNE | BPF_X: case BPF_JMP32 | BPF_JNE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bne(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGT | BPF_X: case BPF_JMP32 | BPF_JSGT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_blt(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLT | BPF_X: case BPF_JMP32 | BPF_JSLT | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_blt(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGE | BPF_X: case BPF_JMP32 | BPF_JSGE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bge(rd, rs, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLE | BPF_X: case BPF_JMP32 | BPF_JSLE | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_sext_32_rd_rs(&rd, &rs, ctx); - emit(rv_bge(rs, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSET | BPF_X: case BPF_JMP32 | BPF_JSET | BPF_X: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - if (!is64) - emit_zext_32_rd_rs(&rd, &rs, ctx); - emit(rv_and(RV_REG_T1, rd, rs), ctx); - emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx); + rvoff = rv_offset(i, off, ctx); + if (!is64) { + s = ctx->ninsns; + if (is_signed_bpf_cond(BPF_OP(code))) + emit_sext_32_rd_rs(&rd, &rs, ctx); + else + emit_zext_32_rd_rs(&rd, &rs, ctx); + e = ctx->ninsns; + + /* Adjust for extra insns */ + rvoff -= (e - s) << 2; + } + + if (BPF_OP(code) == BPF_JSET) { + /* Adjust for and */ + rvoff -= 4; + emit(rv_and(RV_REG_T1, rd, rs), ctx); + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, + ctx); + } else { + emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); + } break; /* IF (dst COND imm) JUMP off */ case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP32 | BPF_JEQ | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_beq(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGT | BPF_K: case BPF_JMP32 | BPF_JGT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bltu(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLT | BPF_K: case BPF_JMP32 | BPF_JLT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bltu(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JGE | BPF_K: case BPF_JMP32 | BPF_JGE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bgeu(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JLE | BPF_K: case BPF_JMP32 | BPF_JLE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bgeu(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JNE | BPF_K: case BPF_JMP32 | BPF_JNE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_bne(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGT | BPF_K: case BPF_JMP32 | BPF_JSGT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_blt(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSLT | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_blt(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSGE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_bge(rd, RV_REG_T1, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSLE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; - emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_sext_32_rd(&rd, ctx); - emit(rv_bge(RV_REG_T1, rd, rvoff >> 1), ctx); - break; case BPF_JMP | BPF_JSET | BPF_K: case BPF_JMP32 | BPF_JSET | BPF_K: - if (rv_offset_check(&rvoff, off, i, ctx)) - return -1; + rvoff = rv_offset(i, off, ctx); + s = ctx->ninsns; emit_imm(RV_REG_T1, imm, ctx); - if (!is64) - emit_zext_32_rd_t1(&rd, ctx); - emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); - emit(rv_bne(RV_REG_T1, RV_REG_ZERO, rvoff >> 1), ctx); + if (!is64) { + if (is_signed_bpf_cond(BPF_OP(code))) + emit_sext_32_rd(&rd, ctx); + else + emit_zext_32_rd_t1(&rd, ctx); + } + e = ctx->ninsns; + + /* Adjust for extra insns */ + rvoff -= (e - s) << 2; + + if (BPF_OP(code) == BPF_JSET) { + /* Adjust for and */ + rvoff -= 4; + emit(rv_and(RV_REG_T1, rd, RV_REG_T1), ctx); + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, + ctx); + } else { + emit_branch(BPF_OP(code), rd, RV_REG_T1, rvoff, ctx); + } break; /* function call */ case BPF_JMP | BPF_CALL: { bool fixed; - int i, ret; + int ret; u64 addr; mark_call(ctx); @@ -1215,20 +1240,9 @@ out_be: &fixed); if (ret < 0) return ret; - if (fixed) { - emit_imm(RV_REG_T1, addr, ctx); - } else { - i = ctx->ninsns; - emit_imm(RV_REG_T1, addr, ctx); - for (i = ctx->ninsns - i; i < 8; i++) { - /* nop */ - emit(rv_addi(RV_REG_ZERO, RV_REG_ZERO, 0), - ctx); - } - } - emit(rv_jalr(RV_REG_RA, RV_REG_T1, 0), ctx); - rd = bpf_to_rv_reg(BPF_REG_0, ctx); - emit(rv_addi(rd, RV_REG_A0, 0), ctx); + ret = emit_call(fixed, addr, ctx); + if (ret) + return ret; break; } /* tail call */ @@ -1243,9 +1257,7 @@ out_be: break; rvoff = epilogue_offset(ctx); - if (is_21b_check(rvoff, i)) - return -1; - emit(rv_jal(RV_REG_ZERO, rvoff >> 1), ctx); + emit_jump_and_link(RV_REG_ZERO, rvoff, false, ctx); break; /* dst = imm64 */ @@ -1426,6 +1438,10 @@ static void build_prologue(struct rv_jit_context *ctx) { int stack_adjust = 0, store_offset, bpf_stack_adjust; + bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); + if (bpf_stack_adjust) + mark_fp(ctx); + if (seen_reg(RV_REG_RA, ctx)) stack_adjust += 8; stack_adjust += 8; /* RV_REG_FP */ @@ -1443,7 +1459,6 @@ static void build_prologue(struct rv_jit_context *ctx) stack_adjust += 8; stack_adjust = round_up(stack_adjust, 16); - bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); stack_adjust += bpf_stack_adjust; store_offset = stack_adjust - 8; @@ -1502,10 +1517,10 @@ static void build_prologue(struct rv_jit_context *ctx) static void build_epilogue(struct rv_jit_context *ctx) { - __build_epilogue(RV_REG_RA, ctx); + __build_epilogue(false, ctx); } -static int build_body(struct rv_jit_context *ctx, bool extra_pass) +static int build_body(struct rv_jit_context *ctx, bool extra_pass, int *offset) { const struct bpf_prog *prog = ctx->prog; int i; @@ -1517,12 +1532,12 @@ static int build_body(struct rv_jit_context *ctx, bool extra_pass) ret = emit_insn(insn, ctx, extra_pass); if (ret > 0) { i++; - if (ctx->insns == NULL) - ctx->offset[i] = ctx->ninsns; + if (offset) + offset[i] = ctx->ninsns; continue; } - if (ctx->insns == NULL) - ctx->offset[i] = ctx->ninsns; + if (offset) + offset[i] = ctx->ninsns; if (ret) return ret; } @@ -1548,9 +1563,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { bool tmp_blinded = false, extra_pass = false; struct bpf_prog *tmp, *orig_prog = prog; + int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; + unsigned int image_size = 0; struct rv_jit_context *ctx; - unsigned int image_size; if (!prog->jit_requested) return orig_prog; @@ -1587,33 +1603,59 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog = orig_prog; goto out_offset; } + for (i = 0; i < prog->len; i++) { + prev_ninsns += 32; + ctx->offset[i] = prev_ninsns; + } - /* First pass generates the ctx->offset, but does not emit an image. */ - if (build_body(ctx, extra_pass)) { - prog = orig_prog; - goto out_offset; + for (i = 0; i < 16; i++) { + pass++; + ctx->ninsns = 0; + if (build_body(ctx, extra_pass, ctx->offset)) { + prog = orig_prog; + goto out_offset; + } + build_prologue(ctx); + ctx->epilogue_offset = ctx->ninsns; + build_epilogue(ctx); + + if (ctx->ninsns == prev_ninsns) { + if (jit_data->header) + break; + + image_size = sizeof(u32) * ctx->ninsns; + jit_data->header = + bpf_jit_binary_alloc(image_size, + &jit_data->image, + sizeof(u32), + bpf_fill_ill_insns); + if (!jit_data->header) { + prog = orig_prog; + goto out_offset; + } + + ctx->insns = (u32 *)jit_data->image; + /* Now, when the image is allocated, the image + * can potentially shrink more (auipc/jalr -> + * jal). + */ + } + prev_ninsns = ctx->ninsns; } - build_prologue(ctx); - ctx->epilogue_offset = ctx->ninsns; - build_epilogue(ctx); - /* Allocate image, now that we know the size. */ - image_size = sizeof(u32) * ctx->ninsns; - jit_data->header = bpf_jit_binary_alloc(image_size, &jit_data->image, - sizeof(u32), - bpf_fill_ill_insns); - if (!jit_data->header) { + if (i == 16) { + pr_err("bpf-jit: image did not converge in <%d passes!\n", i); + bpf_jit_binary_free(jit_data->header); prog = orig_prog; goto out_offset; } - /* Second, real pass, that acutally emits the image. */ - ctx->insns = (u32 *)jit_data->image; skip_init_ctx: + pass++; ctx->ninsns = 0; build_prologue(ctx); - if (build_body(ctx, extra_pass)) { + if (build_body(ctx, extra_pass, NULL)) { bpf_jit_binary_free(jit_data->header); prog = orig_prog; goto out_offset; @@ -1621,7 +1663,7 @@ skip_init_ctx: build_epilogue(ctx); if (bpf_jit_enable > 1) - bpf_jit_dump(prog->len, image_size, 2, ctx->insns); + bpf_jit_dump(prog->len, image_size, pass, ctx->insns); prog->bpf_func = (void *)ctx->insns; prog->jited = 1; @@ -1641,3 +1683,16 @@ out: tmp : orig_prog); return prog; } + +void *bpf_jit_alloc_exec(unsigned long size) +{ + return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, + BPF_JIT_REGION_END, GFP_KERNEL, + PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, + __builtin_return_address(0)); +} + +void bpf_jit_free_exec(void *addr) +{ + return vfree(addr); +} |