diff mbox series

[7/8] bpf/arm64: add atomic-exchange-and-add operation

Message ID 20190903105938.33231-8-jerinj@marvell.com (mailing list archive)
State Accepted, archived
Delegated to: Thomas Monjalon
Headers
Series eBPF arm64 JIT support |

Checks

Context Check Description
ci/checkpatch success coding style OK
ci/Intel-compilation success Compilation OK

Commit Message

Jerin Jacob Kollanukkaran Sept. 3, 2019, 10:59 a.m. UTC 
  From: Jerin Jacob <jerinj@marvell.com>

Implement XADD eBPF instruction using STADD arm64 instruction.
If the given platform does not have atomics support,
use LDXR and STXR pair for critical section instead of STADD.

Signed-off-by: Jerin Jacob <jerinj@marvell.com>
---
 lib/librte_bpf/bpf_jit_arm64.c | 85 +++++++++++++++++++++++++++++++++-
 1 file changed, 84 insertions(+), 1 deletion(-)

Comments

David Marchand Oct. 18, 2019, 1:16 p.m. UTC | #1 
On Tue, Sep 3, 2019 at 1:00 PM <jerinj@marvell.com> wrote:
>
> From: Jerin Jacob <jerinj@marvell.com>
>
> Implement XADD eBPF instruction using STADD arm64 instruction.
> If the given platform does not have atomics support,
> use LDXR and STXR pair for critical section instead of STADD.

For the record, this patch had a missed dependency on the 128-bits
atomic for arm64 patch because of RTE_ARM_FEATURE_ATOMICS.
This will be resolved once the latter is merged.


> ---
>  lib/librte_bpf/bpf_jit_arm64.c | 85 +++++++++++++++++++++++++++++++++-
>  1 file changed, 84 insertions(+), 1 deletion(-)
>
> diff --git a/lib/librte_bpf/bpf_jit_arm64.c b/lib/librte_bpf/bpf_jit_arm64.c
> index c797c9c62..62fa6a505 100644
> --- a/lib/librte_bpf/bpf_jit_arm64.c
> +++ b/lib/librte_bpf/bpf_jit_arm64.c

[snip]

> +static int
> +has_atomics(void)
> +{
> +       int rc = 0;
> +
> +#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
> +       rc = 1;
> +#endif
> +       return rc;
> +}

[snip]
diff mbox series

Patch

diff --git a/lib/librte_bpf/bpf_jit_arm64.c b/lib/librte_bpf/bpf_jit_arm64.c
index c797c9c62..62fa6a505 100644
--- a/lib/librte_bpf/bpf_jit_arm64.c
+++ b/lib/librte_bpf/bpf_jit_arm64.c
@@ -837,6 +837,83 @@  emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src)
 	emit_b(ctx, jump_to_epilogue);
 }
 
+static void
+emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn)
+{
+	uint32_t insn;
+
+	insn = 0xb820001f;
+	insn |= (!!is64) << 30;
+	insn |= rs << 16;
+	insn |= rn << 5;
+
+	emit_insn(ctx, insn, check_reg(rs) || check_reg(rn));
+}
+
+static void
+emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn)
+{
+	uint32_t insn;
+
+	insn = 0x885f7c00;
+	insn |= (!!is64) << 30;
+	insn |= rn << 5;
+	insn |= rt;
+
+	emit_insn(ctx, insn, check_reg(rt) || check_reg(rn));
+}
+
+static void
+emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt,
+	  uint8_t rn)
+{
+	uint32_t insn;
+
+	insn = 0x88007c00;
+	insn |= (!!is64) << 30;
+	insn |= rs << 16;
+	insn |= rn << 5;
+	insn |= rt;
+
+	emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn));
+}
+
+static int
+has_atomics(void)
+{
+	int rc = 0;
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+	rc = 1;
+#endif
+	return rc;
+}
+
+static void
+emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2,
+	  uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src)
+{
+	bool is64 = (BPF_SIZE(op) == EBPF_DW);
+	uint8_t rn;
+
+	if (off) {
+		emit_mov_imm(ctx, 1, tmp1, off);
+		emit_add(ctx, 1, tmp1, dst);
+		rn = tmp1;
+	} else {
+		rn = dst;
+	}
+
+	if (has_atomics()) {
+		emit_stadd(ctx, is64, src, rn);
+	} else {
+		emit_ldxr(ctx, is64, tmp2, rn);
+		emit_add(ctx, is64, tmp2, src);
+		emit_stxr(ctx, is64, tmp3, tmp2, rn);
+		emit_cbnz(ctx, is64, tmp3, -3);
+	}
+}
+
 static void
 check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 {
@@ -863,7 +940,7 @@  check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 static int
 emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 {
-	uint8_t op, dst, src, tmp1, tmp2;
+	uint8_t op, dst, src, tmp1, tmp2, tmp3;
 	const struct ebpf_insn *ins;
 	uint64_t u64;
 	int16_t off;
@@ -878,6 +955,7 @@  emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 	ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16);
 	tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
 	tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
+	tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3);
 
 	emit_prologue(ctx);
 
@@ -1067,6 +1145,11 @@  emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 			emit_mov_imm(ctx, 1, tmp2, off);
 			emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2);
 			break;
+		/* STX XADD: lock *(size *)(dst + off) += src */
+		case (BPF_STX | EBPF_XADD | BPF_W):
+		case (BPF_STX | EBPF_XADD | EBPF_DW):
+			emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src);
+			break;
 		/* Return r0 */
 		case (BPF_JMP | EBPF_EXIT):
 			emit_epilogue(ctx);