From patchwork Wed Jun 7 13:02:36 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Ivan Malov X-Patchwork-Id: 128314 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8A92842C4D; Wed, 7 Jun 2023 15:06:41 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3023242FEC; Wed, 7 Jun 2023 15:03:21 +0200 (CEST) Received: from agw.arknetworks.am (agw.arknetworks.am [79.141.165.80]) by mails.dpdk.org (Postfix) with ESMTP id 16C0742D63 for ; Wed, 7 Jun 2023 15:03:01 +0200 (CEST) Received: from localhost.localdomain (unknown [78.109.69.83]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) by agw.arknetworks.am (Postfix) with ESMTPSA id 9686CE12C4; Wed, 7 Jun 2023 17:03:00 +0400 (+04) From: Ivan Malov To: dev@dpdk.org Cc: Andrew Rybchenko , Ferruh Yigit , Andy Moreton Subject: [PATCH v4 25/34] net/sfc: make use of conntrack assistance for transfer flows Date: Wed, 7 Jun 2023 17:02:36 +0400 Message-Id: <20230607130245.8048-26-ivan.malov@arknetworks.am> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20230607130245.8048-1-ivan.malov@arknetworks.am> References: <20230601195538.8265-1-ivan.malov@arknetworks.am> <20230607130245.8048-1-ivan.malov@arknetworks.am> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org On EF100 hardware, match-action engine (MAE) can be equipped with an assistance table for connection tracking (CT). In it, an entry key is a set of exact match fields: an EtherType, a pair of IP addresses, a L4 protocol ID and a pair of L4 port numbers. An entry response can provide matching packets with a mark value and additional data to be plumbed to NAT action. In addition, an update to mark-and-sweep counter can be done. This table was designed with larger capacity in mind, so moving the above match criteria out of an action rule (AR) specification to a CT entry increases the likelihood of reusing AR entries and improves the total flow engine capacity. Make use of that. NAT and CT counters will be supported later. Signed-off-by: Ivan Malov Reviewed-by: Andy Moreton --- drivers/net/sfc/sfc_flow.h | 4 + drivers/net/sfc/sfc_mae.c | 314 +++++++++++++++++++++++++++++++++++-- drivers/net/sfc/sfc_mae.h | 1 + 3 files changed, 310 insertions(+), 9 deletions(-) diff --git a/drivers/net/sfc/sfc_flow.h b/drivers/net/sfc/sfc_flow.h index 10c73d012f..8f706fc589 100644 --- a/drivers/net/sfc/sfc_flow.h +++ b/drivers/net/sfc/sfc_flow.h @@ -18,6 +18,7 @@ #include "efx.h" #include "sfc_flow_rss.h" +#include "sfc_mae_ct.h" #ifdef __cplusplus extern "C" { @@ -82,6 +83,9 @@ struct sfc_flow_spec_mae { struct sfc_mae_outer_rule *outer_rule; /* Action rule registry entry */ struct sfc_mae_action_rule *action_rule; + /* Conntrack (CT) assistance table entry key and response */ + sfc_mae_conntrack_response_t ct_resp; + sfc_mae_conntrack_key_t ct_key; }; /* Flow specification */ diff --git a/drivers/net/sfc/sfc_mae.c b/drivers/net/sfc/sfc_mae.c index addcad2843..d3b4099213 100644 --- a/drivers/net/sfc/sfc_mae.c +++ b/drivers/net/sfc/sfc_mae.c @@ -18,6 +18,7 @@ #include "sfc.h" #include "sfc_flow_tunnel.h" #include "sfc_mae_counter.h" +#include "sfc_mae_ct.h" #include "sfc_log.h" #include "sfc_switch.h" #include "sfc_service.h" @@ -1177,18 +1178,23 @@ struct sfc_mae_action_rule_ctx { struct sfc_mae_outer_rule *outer_rule; struct sfc_mae_action_set *action_set; efx_mae_match_spec_t *match_spec; + uint32_t ct_mark; }; static int sfc_mae_action_rule_attach(struct sfc_adapter *sa, - const struct sfc_mae_action_rule_ctx *ctx, + struct sfc_mae_action_rule_ctx *ctx, struct sfc_mae_action_rule **rulep, - __rte_unused struct rte_flow_error *error) + struct rte_flow_error *error) { + uint32_t new_ct_mark = ctx->ct_mark; struct sfc_mae_action_rule *rule; + int rc; SFC_ASSERT(sfc_adapter_is_locked(sa)); + SFC_ASSERT(ctx->ct_mark <= 1); + /* * It is assumed that the caller of this helper has already properly * tailored ctx->match_spec to match on OR_ID / 0xffffffff (when @@ -1196,10 +1202,24 @@ sfc_mae_action_rule_attach(struct sfc_adapter *sa, * on 0xffffffff / 0xffffffff, so that specs compare correctly. */ TAILQ_FOREACH(rule, &sa->mae.action_rules, entries) { + if (rule->ct_mark == new_ct_mark) + ++new_ct_mark; + if (rule->outer_rule != ctx->outer_rule || - rule->action_set != ctx->action_set) + rule->action_set != ctx->action_set || + !!rule->ct_mark != !!ctx->ct_mark) continue; + if (ctx->ct_mark != 0) { + rc = efx_mae_match_spec_ct_mark_set(ctx->match_spec, + rule->ct_mark); + if (rc != 0) { + return rte_flow_error_set(error, EFAULT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "AR: failed to set CT mark for comparison"); + } + } + if (efx_mae_match_specs_equal(rule->match_spec, ctx->match_spec)) { sfc_dbg(sa, "attaching to action_rule=%p", rule); @@ -1209,6 +1229,24 @@ sfc_mae_action_rule_attach(struct sfc_adapter *sa, } } + if (ctx->ct_mark != 0) { + if (new_ct_mark == UINT32_MAX) { + return rte_flow_error_set(error, ERANGE, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "AR: failed to allocate CT mark"); + } + + rc = efx_mae_match_spec_ct_mark_set(ctx->match_spec, + new_ct_mark); + if (rc != 0) { + return rte_flow_error_set(error, EFAULT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "AR: failed to set CT mark"); + } + + ctx->ct_mark = new_ct_mark; + } + /* * No need to set RTE error, as this * code should be handled gracefully. @@ -1231,6 +1269,17 @@ sfc_mae_action_rule_add(struct sfc_adapter *sa, return ENOMEM; rule->refcnt = 1; + + /* + * It is assumed that the caller invoked sfc_mae_action_rule_attach() + * and got (-ENOENT) before getting here. That ensures a unique CT + * mark value or, if no CT is involved at all, simply zero. + * + * It is also assumed that match on the mark (if non-zero) + * is already set in the action rule match specification. + */ + rule->ct_mark = ctx->ct_mark; + rule->outer_rule = ctx->outer_rule; rule->action_set = ctx->action_set; rule->match_spec = ctx->match_spec; @@ -1802,6 +1851,8 @@ struct sfc_mae_field_locator { size_t size; /* Field offset in the corresponding rte_flow_item_ struct */ size_t ofst; + + uint8_t ct_key_field; }; static void @@ -2654,6 +2705,216 @@ static const struct sfc_flow_item sfc_flow_items[] = { }, }; +#define SFC_MAE_CT_KEY_ET 0x01 /* EtherType */ +#define SFC_MAE_CT_KEY_DA 0x02 /* IPv4/IPv6 destination address */ +#define SFC_MAE_CT_KEY_SA 0x04 /* IPv4/IPv6 source address */ +#define SFC_MAE_CT_KEY_L4 0x08 /* IPv4/IPv6 L4 protocol ID */ +#define SFC_MAE_CT_KEY_DP 0x10 /* L4 destination port */ +#define SFC_MAE_CT_KEY_SP 0x20 /* L4 source port */ + +#define SFC_MAE_CT_KEY_FIELD_SIZE_MAX sizeof(sfc_mae_conntrack_key_t) + +static const struct sfc_mae_field_locator flocs_ct[] = { + { + EFX_MAE_FIELD_ETHER_TYPE_BE, + RTE_SIZEOF_FIELD(sfc_mae_conntrack_key_t, ether_type_le), + offsetof(sfc_mae_conntrack_key_t, ether_type_le), + SFC_MAE_CT_KEY_ET, + }, + { + EFX_MAE_FIELD_DST_IP4_BE, + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.dst_addr), + offsetof(sfc_mae_conntrack_key_t, dst_addr_le) + + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.dst_addr) - + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.dst_addr), + SFC_MAE_CT_KEY_DA, + }, + { + EFX_MAE_FIELD_SRC_IP4_BE, + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.src_addr), + offsetof(sfc_mae_conntrack_key_t, src_addr_le) + + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.src_addr) - + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv4, hdr.src_addr), + SFC_MAE_CT_KEY_SA, + }, + { + EFX_MAE_FIELD_DST_IP6_BE, + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.dst_addr), + offsetof(sfc_mae_conntrack_key_t, dst_addr_le), + SFC_MAE_CT_KEY_DA, + }, + { + EFX_MAE_FIELD_SRC_IP6_BE, + RTE_SIZEOF_FIELD(struct rte_flow_item_ipv6, hdr.src_addr), + offsetof(sfc_mae_conntrack_key_t, src_addr_le), + SFC_MAE_CT_KEY_SA, + }, + { + EFX_MAE_FIELD_IP_PROTO, + RTE_SIZEOF_FIELD(sfc_mae_conntrack_key_t, ip_proto), + offsetof(sfc_mae_conntrack_key_t, ip_proto), + SFC_MAE_CT_KEY_L4, + }, + { + EFX_MAE_FIELD_L4_DPORT_BE, + RTE_SIZEOF_FIELD(sfc_mae_conntrack_key_t, dst_port_le), + offsetof(sfc_mae_conntrack_key_t, dst_port_le), + SFC_MAE_CT_KEY_DP, + }, + { + EFX_MAE_FIELD_L4_SPORT_BE, + RTE_SIZEOF_FIELD(sfc_mae_conntrack_key_t, src_port_le), + offsetof(sfc_mae_conntrack_key_t, src_port_le), + SFC_MAE_CT_KEY_SP, + }, +}; + +static int +sfc_mae_rule_process_ct(struct sfc_adapter *sa, struct sfc_mae_parse_ctx *pctx, + struct sfc_mae_action_rule_ctx *action_rule_ctx, + struct sfc_flow_spec_mae *spec, + struct rte_flow_error *error) +{ + efx_mae_match_spec_t *match_spec_tmp; + uint8_t ct_key_missing_fields = + SFC_MAE_CT_KEY_ET | SFC_MAE_CT_KEY_DA | SFC_MAE_CT_KEY_SA | + SFC_MAE_CT_KEY_L4 | SFC_MAE_CT_KEY_DP | SFC_MAE_CT_KEY_SP; + unsigned int i; + int rc; + + if (pctx->ft_rule_type == SFC_FT_RULE_TUNNEL) { + /* + * TUNNEL rules have no network match fields that belong + * in an action rule match specification, so nothing can + * be possibly utilised for conntrack assistance offload. + */ + return 0; + } + + if (!sfc_mae_conntrack_is_supported(sa)) + return 0; + + rc = efx_mae_match_spec_clone(sa->nic, pctx->match_spec_action, + &match_spec_tmp); + if (rc != 0) { + return rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "AR: failed to clone the match specification"); + } + + for (i = 0; i < RTE_DIM(flocs_ct); ++i) { + const struct sfc_mae_field_locator *fl = &flocs_ct[i]; + uint8_t mask_full[SFC_MAE_CT_KEY_FIELD_SIZE_MAX]; + uint8_t mask_zero[SFC_MAE_CT_KEY_FIELD_SIZE_MAX]; + uint8_t value[SFC_MAE_CT_KEY_FIELD_SIZE_MAX]; + uint8_t mask[SFC_MAE_CT_KEY_FIELD_SIZE_MAX]; + uint8_t *ct_key = (uint8_t *)&spec->ct_key; + efx_mae_field_id_t fid = fl->field_id; + unsigned int j; + + rc = efx_mae_match_spec_field_get(match_spec_tmp, fid, + fl->size, value, + fl->size, mask); + if (rc != 0) { + return rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "AR: failed to extract match field"); + } + + memset(mask_full, 0xff, fl->size); + + if (memcmp(mask, mask_full, fl->size) != 0) + continue; + + memset(mask_zero, 0, fl->size); + + rc = efx_mae_match_spec_field_set(match_spec_tmp, fid, + fl->size, mask_zero, + fl->size, mask_zero); + if (rc != 0) { + return rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "AR: failed to erase match field"); + } + + for (j = 0; j < fl->size; ++j) { + uint8_t *byte_dst = ct_key + fl->ofst + fl->size - 1 - j; + const uint8_t *byte_src = value + j; + + *byte_dst = *byte_src; + } + + ct_key_missing_fields &= ~(fl->ct_key_field); + } + + if (ct_key_missing_fields != 0) { + efx_mae_match_spec_fini(sa->nic, match_spec_tmp); + return 0; + } + + efx_mae_match_spec_fini(sa->nic, pctx->match_spec_action); + pctx->match_spec_action = match_spec_tmp; + + if (pctx->ft_rule_type == SFC_FT_RULE_SWITCH) { + /* + * A SWITCH rule re-uses the corresponding TUNNEL rule's + * outer rule, where conntrack request should have been + * configured already, so skip outer rule processing. + */ + goto skip_outer_rule; + } + + if (pctx->match_spec_outer == NULL) { + const struct sfc_mae_pattern_data *pdata = &pctx->pattern_data; + const struct sfc_mae_ethertype *et; + struct sfc_mae *mae = &sa->mae; + + rc = efx_mae_match_spec_init(sa->nic, + EFX_MAE_RULE_OUTER, + mae->nb_outer_rule_prios_max - 1, + &pctx->match_spec_outer); + if (rc != 0) { + return rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "OR: failed to initialise the match specification"); + } + + /* Match on EtherType appears to be compulsory in outer rules */ + + et = &pdata->ethertypes[pdata->nb_vlan_tags]; + + rc = efx_mae_match_spec_field_set(pctx->match_spec_outer, + EFX_MAE_FIELD_ENC_ETHER_TYPE_BE, + sizeof(et->value), (const uint8_t *)&et->value, + sizeof(et->mask), (const uint8_t *)&et->mask); + if (rc != 0) { + return rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "OR: failed to set match on EtherType"); + } + } + + rc = efx_mae_outer_rule_do_ct_set(pctx->match_spec_outer); + if (rc != 0) { + return rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "OR: failed to request CT lookup"); + } + +skip_outer_rule: + /* Initial/dummy CT mark value */ + action_rule_ctx->ct_mark = 1; + + return 0; +} + +#undef SFC_MAE_CT_KEY_ET +#undef SFC_MAE_CT_KEY_DA +#undef SFC_MAE_CT_KEY_SA +#undef SFC_MAE_CT_KEY_L4 +#undef SFC_MAE_CT_KEY_DP +#undef SFC_MAE_CT_KEY_SP + static int sfc_mae_rule_process_outer(struct sfc_adapter *sa, struct sfc_mae_parse_ctx *ctx, @@ -2677,13 +2938,11 @@ sfc_mae_rule_process_outer(struct sfc_adapter *sa, return 0; } - if (ctx->encap_type == EFX_TUNNEL_PROTOCOL_NONE) { + if (ctx->match_spec_outer == NULL) { *rulep = NULL; goto no_or_id; } - SFC_ASSERT(ctx->match_spec_outer != NULL); - if (!efx_mae_match_spec_is_valid(sa->nic, ctx->match_spec_outer)) { return rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, NULL, @@ -2810,6 +3069,7 @@ sfc_mae_rule_encap_parse_init(struct sfc_adapter *sa, { const struct rte_flow_item *pattern = ctx->pattern; struct sfc_mae *mae = &sa->mae; + bool request_ct = false; uint8_t recirc_id = 0; int rc; @@ -2905,6 +3165,14 @@ sfc_mae_rule_encap_parse_init(struct sfc_adapter *sa, switch (ctx->ft_rule_type) { case SFC_FT_RULE_TUNNEL: recirc_id = SFC_FT_CTX_ID_TO_CTX_MARK(ctx->ft_ctx->id); + + if (sfc_mae_conntrack_is_supported(sa)) { + /* + * Request lookup in conntrack table since SWITCH rules + * are eligible to utilise this type of assistance. + */ + request_ct = true; + } /* FALLTHROUGH */ case SFC_FT_RULE_NONE: if (ctx->priority >= mae->nb_outer_rule_prios_max) { @@ -2938,6 +3206,16 @@ sfc_mae_rule_encap_parse_init(struct sfc_adapter *sa, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "OR: failed to initialise RECIRC_ID"); } + + if (!request_ct) + break; + + rc = efx_mae_outer_rule_do_ct_set(ctx->match_spec_outer); + if (rc != 0) { + return rte_flow_error_set(error, rc, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "OR: failed to request CT lookup"); + } break; case SFC_FT_RULE_SWITCH: /* Outermost items -> "ENC" match fields in the action rule. */ @@ -2959,9 +3237,6 @@ static void sfc_mae_rule_encap_parse_fini(struct sfc_adapter *sa, struct sfc_mae_parse_ctx *ctx) { - if (ctx->encap_type == EFX_TUNNEL_PROTOCOL_NONE) - return; - if (ctx->match_spec_outer != NULL) efx_mae_match_spec_fini(sa->nic, ctx->match_spec_outer); } @@ -3065,6 +3340,11 @@ sfc_mae_rule_parse_pattern(struct sfc_adapter *sa, if (rc != 0) goto fail_process_pattern_data; + rc = sfc_mae_rule_process_ct(sa, &ctx_mae, action_rule_ctx, + spec, error); + if (rc != 0) + goto fail_process_ct; + rc = sfc_mae_rule_process_outer(sa, &ctx_mae, outer_rulep, error); if (rc != 0) goto fail_process_outer; @@ -3083,6 +3363,7 @@ sfc_mae_rule_parse_pattern(struct sfc_adapter *sa, fail_validate_match_spec_action: fail_process_outer: +fail_process_ct: fail_process_pattern_data: fail_parse_pattern: sfc_mae_rule_encap_parse_fini(sa, &ctx_mae); @@ -4380,6 +4661,18 @@ sfc_mae_flow_insert(struct sfc_adapter *sa, if (rc != 0) return rc; + if (spec_mae->action_rule->ct_mark != 0) { + spec_mae->ct_resp.ct_mark = spec_mae->action_rule->ct_mark; + spec_mae->ct_resp.counter_id = EFX_MAE_RSRC_ID_INVALID; + + rc = sfc_mae_conntrack_insert(sa, &spec_mae->ct_key, + &spec_mae->ct_resp); + if (rc != 0) { + sfc_mae_action_rule_disable(sa, action_rule); + return rc; + } + } + return 0; } @@ -4394,6 +4687,9 @@ sfc_mae_flow_remove(struct sfc_adapter *sa, if (action_rule == NULL) return 0; + if (action_rule->ct_mark != 0) + (void)sfc_mae_conntrack_delete(sa, &spec_mae->ct_key); + sfc_mae_action_rule_disable(sa, action_rule); return 0; diff --git a/drivers/net/sfc/sfc_mae.h b/drivers/net/sfc/sfc_mae.h index cbe190075c..67fa2ca5c9 100644 --- a/drivers/net/sfc/sfc_mae.h +++ b/drivers/net/sfc/sfc_mae.h @@ -100,6 +100,7 @@ TAILQ_HEAD(sfc_mae_action_sets, sfc_mae_action_set); /** Action rule registry entry */ struct sfc_mae_action_rule { TAILQ_ENTRY(sfc_mae_action_rule) entries; + uint32_t ct_mark; struct sfc_mae_outer_rule *outer_rule; struct sfc_mae_action_set *action_set; efx_mae_match_spec_t *match_spec;