From 0fcd593b943bfcc21ad84d3321422401de071d8a Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Thu, 12 Nov 2015 13:57:00 -0800 Subject: arm64: bpf: fix JIT frame pointer setup BPF fp should point to the top of the BPF prog stack. The original implementation made it point to the bottom incorrectly. Move A64_SP to fp before reserve BPF prog stack space. CC: Zi Shen Lim CC: Xi Wang Signed-off-by: Yang Shi Reviewed-by: Zi Shen Lim Signed-off-by: David S. Miller --- arch/arm64/net/bpf_jit_comp.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/arm64') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index a44e5293c6f5..ac8b548ce493 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -161,12 +161,12 @@ static void build_prologue(struct jit_ctx *ctx) if (ctx->tmp_used) emit(A64_PUSH(tmp1, tmp2, A64_SP), ctx); - /* Set up BPF stack */ - emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx); - /* Set up frame pointer */ emit(A64_MOV(1, fp, A64_SP), ctx); + /* Set up BPF stack */ + emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx); + /* Clear registers A and X */ emit_a64_mov_i64(ra, 0, ctx); emit_a64_mov_i64(rx, 0, ctx); -- cgit v1.2.3 From c3d4c682c240595c3637c552dc1afa985bc6d382 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Sat, 14 Nov 2015 01:16:18 +0100 Subject: bpf, arm64: start flushing icache range from header While recently going over ARM64's BPF code, I noticed that the icache range we're flushing should start at header already and not at ctx.image. Reason is that after b569c1c622c5 ("net: bpf: arm64: address randomize and write protect JIT code"), we also want to make sure to flush the random-sized trap in front of the start of the actual program (analogous to x86). No operational differences from user side. Signed-off-by: Daniel Borkmann Acked-by: Zi Shen Lim Cc: Alexei Starovoitov Signed-off-by: David S. Miller --- arch/arm64/net/bpf_jit_comp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm64') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index ac8b548ce493..64a8bc12f4a2 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -740,7 +740,7 @@ void bpf_int_jit_compile(struct bpf_prog *prog) if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, image_size, 2, ctx.image); - bpf_flush_icache(ctx.image, ctx.image + ctx.idx); + bpf_flush_icache(header, ctx.image + ctx.idx); set_memory_ro((unsigned long)header, header->pages); prog->bpf_func = (void *)ctx.image; -- cgit v1.2.3 From ec0738db8d7a9bd6653d9f9d81b201a519144d63 Mon Sep 17 00:00:00 2001 From: Yang Shi Date: Mon, 16 Nov 2015 14:35:35 -0800 Subject: arm64: bpf: make BPF prologue and epilogue align with ARM64 AAPCS Save and restore FP/LR in BPF prog prologue and epilogue, save SP to FP in prologue in order to get the correct stack backtrace. However, ARM64 JIT used FP (x29) as eBPF fp register, FP is subjected to change during function call so it may cause the BPF prog stack base address change too. Use x25 to replace FP as BPF stack base register (fp). Since x25 is callee saved register, so it will keep intact during function call. It is initialized in BPF prog prologue when BPF prog is started to run everytime. Save and restore x25/x26 in BPF prologue and epilogue to keep them intact for the outside of BPF. Actually, x26 is unnecessary, but SP requires 16 bytes alignment. So, the BPF stack layout looks like: high original A64_SP => 0:+-----+ BPF prologue |FP/LR| current A64_FP => -16:+-----+ | ... | callee saved registers +-----+ | | x25/x26 BPF fp register => -80:+-----+ | | | ... | BPF prog stack | | | | current A64_SP => +-----+ | | | ... | Function call stack | | +-----+ low CC: Zi Shen Lim CC: Xi Wang Signed-off-by: Yang Shi Acked-by: Zi Shen Lim Signed-off-by: David S. Miller --- arch/arm64/net/bpf_jit_comp.c | 44 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) (limited to 'arch/arm64') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 64a8bc12f4a2..a6905b8d12d3 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -50,7 +50,7 @@ static const int bpf2a64[] = { [BPF_REG_8] = A64_R(21), [BPF_REG_9] = A64_R(22), /* read-only frame pointer to access stack */ - [BPF_REG_FP] = A64_FP, + [BPF_REG_FP] = A64_R(25), /* temporary register for internal BPF JIT */ [TMP_REG_1] = A64_R(23), [TMP_REG_2] = A64_R(24), @@ -155,16 +155,47 @@ static void build_prologue(struct jit_ctx *ctx) stack_size += 4; /* extra for skb_copy_bits buffer */ stack_size = STACK_ALIGN(stack_size); + /* + * BPF prog stack layout + * + * high + * original A64_SP => 0:+-----+ BPF prologue + * |FP/LR| + * current A64_FP => -16:+-----+ + * | ... | callee saved registers + * +-----+ + * | | x25/x26 + * BPF fp register => -80:+-----+ + * | | + * | ... | BPF prog stack + * | | + * | | + * current A64_SP => +-----+ + * | | + * | ... | Function call stack + * | | + * +-----+ + * low + * + */ + + /* Save FP and LR registers to stay align with ARM64 AAPCS */ + emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx); + emit(A64_MOV(1, A64_FP, A64_SP), ctx); + /* Save callee-saved register */ emit(A64_PUSH(r6, r7, A64_SP), ctx); emit(A64_PUSH(r8, r9, A64_SP), ctx); if (ctx->tmp_used) emit(A64_PUSH(tmp1, tmp2, A64_SP), ctx); - /* Set up frame pointer */ + /* Save fp (x25) and x26. SP requires 16 bytes alignment */ + emit(A64_PUSH(fp, A64_R(26), A64_SP), ctx); + + /* Set up BPF prog stack base register (x25) */ emit(A64_MOV(1, fp, A64_SP), ctx); - /* Set up BPF stack */ + /* Set up function call stack */ emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx); /* Clear registers A and X */ @@ -190,14 +221,17 @@ static void build_epilogue(struct jit_ctx *ctx) /* We're done with BPF stack */ emit(A64_ADD_I(1, A64_SP, A64_SP, stack_size), ctx); + /* Restore fs (x25) and x26 */ + emit(A64_POP(fp, A64_R(26), A64_SP), ctx); + /* Restore callee-saved register */ if (ctx->tmp_used) emit(A64_POP(tmp1, tmp2, A64_SP), ctx); emit(A64_POP(r8, r9, A64_SP), ctx); emit(A64_POP(r6, r7, A64_SP), ctx); - /* Restore frame pointer */ - emit(A64_MOV(1, fp, A64_SP), ctx); + /* Restore FP/LR registers */ + emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx); /* Set return value */ emit(A64_MOV(1, A64_R(0), r0), ctx); -- cgit v1.2.3