From patchwork Tue Dec 15 07:31:12 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85187 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id F147AA09E9; Tue, 15 Dec 2020 08:31:50 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 5E897C9B0; Tue, 15 Dec 2020 08:31:37 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id C1ED4C9AE for ; Tue, 15 Dec 2020 08:31:35 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:31 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkD005436; Tue, 15 Dec 2020 09:31:31 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com Date: Tue, 15 Dec 2020 09:31:12 +0200 Message-Id: <20201215073119.404947-2-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 1/8] net/mlx5: use mask for meter register setting X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" ASO meter feature may require to locate the flow context tag action after the ASO action. The register used for the meter tag is also used for the meter color: Bits[0-7] A meter color value set by the HW. Bits[8-31] A flow meter context set by SW. Currently the tag action for meter writes all the bits of the meter register, so it will potentially overwrite meter color when ASO meter action is before the tag action. Set only 24-MSB-bits of meter register in the meter tag action. Signed-off-by: Li Zhang --- drivers/net/mlx5/mlx5_flow.c | 22 ++++++++++++++++------ drivers/net/mlx5/mlx5_flow.h | 2 ++ drivers/net/mlx5/mlx5_flow_dv.c | 2 ++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 6ea9d87d18..1cfa817ae9 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -4161,9 +4161,11 @@ flow_hairpin_split(struct rte_eth_dev *dev, rte_memcpy(actions_rx, actions, sizeof(struct rte_flow_action)); actions_rx++; set_tag = (void *)actions_rx; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL); + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = mlx5_flow_get_reg_id(dev, MLX5_HAIRPIN_RX, 0, NULL), + .data = flow_id, + }; MLX5_ASSERT(set_tag->id > REG_NON); - set_tag->data = flow_id; tag_action->conf = set_tag; /* Create Tx item list. */ rte_memcpy(actions_tx, actions, sizeof(struct rte_flow_action)); @@ -4296,6 +4298,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, struct mlx5_rte_flow_item_tag *tag_spec; struct mlx5_rte_flow_item_tag *tag_mask; uint32_t tag_id = 0; + uint32_t reg_id = 0; bool copy_vlan = false; /* Prepare the actions for prefix and suffix flow. */ @@ -4343,7 +4346,7 @@ flow_meter_split_prep(struct rte_eth_dev *dev, actions_pre++; /* Set the tag. */ set_tag = (void *)actions_pre; - set_tag->id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); + reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); mlx5_ipool_malloc(priv->sh->ipool[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); if (tag_id >= (1 << (sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS))) { @@ -4355,7 +4358,12 @@ flow_meter_split_prep(struct rte_eth_dev *dev, } else if (!tag_id) { return 0; } - set_tag->data = tag_id << MLX5_MTR_COLOR_BITS; + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = reg_id, + .offset = MLX5_MTR_COLOR_BITS, + .length = sizeof(tag_id) * 8 - MLX5_MTR_COLOR_BITS, + .data = tag_id, + }; assert(tag_action); tag_action->conf = set_tag; /* Prepare the suffix subflow items. */ @@ -4773,10 +4781,12 @@ flow_sample_split_prep(struct rte_eth_dev *dev, ret = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, 0, error); if (ret < 0) return ret; - set_tag->id = ret; mlx5_ipool_malloc(priv->sh->ipool [MLX5_IPOOL_RSS_EXPANTION_FLOW_ID], &tag_id); - set_tag->data = tag_id; + *set_tag = (struct mlx5_rte_flow_action_set_tag) { + .id = ret, + .data = tag_id, + }; /* Prepare the suffix subflow items. */ tag_spec = (void *)(sfx_items + SAMPLE_SUFFIX_ITEM); tag_spec->data = tag_id; diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 66e19b627b..6bb7bae1bf 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -55,6 +55,8 @@ struct mlx5_rte_flow_item_tag { /* Modify selected register. */ struct mlx5_rte_flow_action_set_tag { enum modify_reg id; + uint8_t offset; + uint8_t length; uint32_t data; }; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 4e5ca4f6c9..7905923665 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -1015,6 +1015,8 @@ flow_dv_convert_action_set_reg actions[i] = (struct mlx5_modification_cmd) { .action_type = MLX5_MODIFICATION_TYPE_SET, .field = reg_to_field[conf->id], + .offset = conf->offset, + .length = conf->length, }; actions[i].data0 = rte_cpu_to_be_32(actions[i].data0); actions[i].data1 = rte_cpu_to_be_32(conf->data); From patchwork Tue Dec 15 07:31:13 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85188 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id B04D4A09E9; Tue, 15 Dec 2020 08:32:10 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 0BE74C9BC; Tue, 15 Dec 2020 08:31:42 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id D3510C9BC for ; Tue, 15 Dec 2020 08:31:40 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:35 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkE005436; Tue, 15 Dec 2020 09:31:35 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com Date: Tue, 15 Dec 2020 09:31:13 +0200 Message-Id: <20201215073119.404947-3-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 2/8] common/mlx5: add definitions for ASO flow meter X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch adds different PRM definitions, related to ASO flow meter feature, in MLX5 PMD code. Signed-off-by: Li Zhang --- drivers/common/mlx5/mlx5_prm.h | 75 ++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h index 48fe54625f..de85b05066 100644 --- a/drivers/common/mlx5/mlx5_prm.h +++ b/drivers/common/mlx5/mlx5_prm.h @@ -1066,6 +1066,8 @@ enum { (1ULL << MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH) #define MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO \ (1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO) +#define MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO \ + (1ULL << MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO) #define MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT \ (1ULL << MLX5_OBJ_TYPE_GENEVE_TLV_OPT) @@ -1418,7 +1420,15 @@ struct mlx5_ifc_qos_cap_bits { u8 reserved_at_c0[0x10]; u8 max_qos_para_vport[0x10]; u8 max_tsar_bw_share[0x20]; - u8 reserved_at_100[0x6e8]; + u8 nic_element_type[0x10]; + u8 nic_tsar_type[0x10]; + u8 reserved_at_120[0x3]; + u8 log_meter_aso_granularity[0x5]; + u8 reserved_at_128[0x3]; + u8 log_meter_aso_max_alloc[0x5]; + u8 reserved_at_130[0x3]; + u8 log_max_num_meter_aso[0x5]; + u8 reserved_at_138[0x6b0]; }; struct mlx5_ifc_per_protocol_networking_offload_caps_bits { @@ -2159,6 +2169,8 @@ struct mlx5_ifc_flow_meter_parameters_bits { u8 eir_mantissa[0x8]; u8 reserved_at_8[0x60]; // 14h-1Ch }; +#define MLX5_IFC_FLOW_METER_PARAM_MASK UINT64_C(0x80FFFFFF) +#define MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL 0x14BF00C8 enum { MLX5_CQE_SIZE_64B = 0x0, @@ -2243,6 +2255,7 @@ enum { MLX5_GENERAL_OBJ_TYPE_VIRTQ = 0x000d, MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c, MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH = 0x0022, + MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO = 0x0024, MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO = 0x0025, }; @@ -2251,7 +2264,9 @@ struct mlx5_ifc_general_obj_in_cmd_hdr_bits { u8 reserved_at_10[0x20]; u8 obj_type[0x10]; u8 obj_id[0x20]; - u8 reserved_at_60[0x20]; + u8 reserved_at_60[0x3]; + u8 log_obj_range[0x5]; + u8 reserved_at_58[0x18]; }; struct mlx5_ifc_general_obj_out_cmd_hdr_bits { @@ -2393,6 +2408,18 @@ struct mlx5_ifc_create_flow_hit_aso_in_bits { struct mlx5_ifc_flow_hit_aso_bits flow_hit_aso; }; +struct mlx5_ifc_flow_meter_aso_bits { + u8 modify_field_select[0x40]; + u8 reserved_at_40[0x48]; + u8 access_pd[0x18]; + u8 reserved_at_a0[0x160]; + u8 parameters[0x200]; +}; + +struct mlx5_ifc_create_flow_meter_aso_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr; + struct mlx5_ifc_flow_meter_aso_bits flow_meter_aso; +}; enum mlx5_access_aso_opc_mod { ASO_OPC_MOD_IPSEC = 0x0, ASO_OPC_MOD_CONNECTION_TRACKING = 0x1, @@ -2446,11 +2473,51 @@ struct mlx5_aso_cseg { uint64_t data_mask; } __rte_packed; +/* A meter data segment - 2 per ASO WQE. */ +struct mlx5_aso_mtr_dseg { + uint32_t v_bo_sc_bbog_mm; + /* + * bit 31: valid, 30: bucket overflow, 28-29: start color, + * 27: both buckets on green, 24-25: meter mode. + */ + uint32_t reserved; + uint32_t cbs_cir; + /* + * bit 24-28: cbs_exponent, bit 16-23 cbs_mantissa, + * bit 8-12: cir_exponent, bit 0-7 cir_mantissa. + */ + uint32_t c_tokens; + uint32_t ebs_eir; + /* + * bit 24-28: ebs_exponent, bit 16-23 ebs_mantissa, + * bit 8-12: eir_exponent, bit 0-7 eir_mantissa. + */ + uint32_t e_tokens; + uint64_t timestamp; +} __rte_packed; + +#define ASO_DSEG_VALID_OFFSET 31 +#define ASO_DSEG_BO_OFFSET 30 +#define ASO_DSEG_SC_OFFSET 28 +#define ASO_DSEG_CBS_EXP_OFFSET 24 +#define ASO_DSEG_CBS_MAN_OFFSET 16 +#define ASO_DSEG_CIR_EXP_MASK 0x1F +#define ASO_DSEG_CIR_EXP_OFFSET 8 +#define ASO_DSEG_EBS_EXP_OFFSET 24 +#define ASO_DSEG_EBS_MAN_OFFSET 16 +#define ASO_DSEG_EXP_MASK 0x1F +#define ASO_DSEG_MAN_MASK 0xFF + #define MLX5_ASO_WQE_DSEG_SIZE 0x40 +#define MLX5_ASO_METERS_PER_WQE 2 +#define MLX5_ASO_MTRS_PER_POOL 128 -/* ASO WQE Data segment. */ +/* ASO WQE data segment. */ struct mlx5_aso_dseg { - uint8_t data[MLX5_ASO_WQE_DSEG_SIZE]; + union { + uint8_t data[MLX5_ASO_WQE_DSEG_SIZE]; + struct mlx5_aso_mtr_dseg mtrs[MLX5_ASO_METERS_PER_WQE]; + }; } __rte_packed; /* ASO WQE. */ From patchwork Tue Dec 15 07:31:14 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85194 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4150AA09E9; Tue, 15 Dec 2020 08:34:04 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 5EB77CA0C; Tue, 15 Dec 2020 08:31:55 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id EBFF5C9CC for ; Tue, 15 Dec 2020 08:31:45 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:41 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkF005436; Tue, 15 Dec 2020 09:31:41 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com Date: Tue, 15 Dec 2020 09:31:14 +0200 Message-Id: <20201215073119.404947-4-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 3/8] common/mlx5: add read ASO flow meter HCA capability X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Read and store the device capability of FLOW_METER_ASO general object, using the DevX API. Signed-off-by: Li Zhang --- drivers/common/mlx5/mlx5_devx_cmds.c | 14 ++++++++++++++ drivers/common/mlx5/mlx5_devx_cmds.h | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index 485363192d..765553ed6b 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -698,6 +698,9 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, attr->max_geneve_tlv_option_data_len = MLX5_GET(cmd_hca_cap, hcattr, max_geneve_tlv_option_data_len); attr->qos.sup = MLX5_GET(cmd_hca_cap, hcattr, qos); + attr->qos.flow_meter_aso_sup = !!(MLX5_GET64(cmd_hca_cap, hcattr, + general_obj_types) & + MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO); attr->vdpa.valid = !!(MLX5_GET64(cmd_hca_cap, hcattr, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q); @@ -756,6 +759,17 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, MLX5_GET(qos_cap, hcattr, packet_pacing); attr->qos.wqe_rate_pp = MLX5_GET(qos_cap, hcattr, wqe_rate_pp); + if (attr->qos.flow_meter_aso_sup) { + attr->qos.log_meter_aso_granularity = + MLX5_GET(qos_cap, hcattr, + log_meter_aso_granularity); + attr->qos.log_meter_aso_max_alloc = + MLX5_GET(qos_cap, hcattr, + log_meter_aso_max_alloc); + attr->qos.log_max_num_meter_aso = + MLX5_GET(qos_cap, hcattr, + log_max_num_meter_aso); + } } if (attr->vdpa.valid) mlx5_devx_cmd_query_hca_vdpa_attr(ctx, &attr->vdpa); diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h index e0638f2064..13338c0e39 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.h +++ b/drivers/common/mlx5/mlx5_devx_cmds.h @@ -42,10 +42,18 @@ struct mlx5_hca_qos_attr { uint32_t wqe_rate_pp:1; /* Packet pacing WQE rate mode. */ uint32_t flow_meter_reg_share:1; /* Whether reg_c share is supported. */ + uint32_t flow_meter_aso_sup:1; + /* Whether FLOW_METER_ASO Object is supported. */ uint8_t log_max_flow_meter; /* Power of the maximum supported meters. */ uint8_t flow_meter_reg_c_ids; /* Bitmap of the reg_Cs available for flow meter to use. */ + uint32_t log_meter_aso_granularity:5; + /* Power of the minimum allocation granularity Object. */ + uint32_t log_meter_aso_max_alloc:5; + /* Power of the maximum allocation granularity Object. */ + uint32_t log_max_num_meter_aso:5; + /* Power of the maximum number of supported objects. */ }; From patchwork Tue Dec 15 07:31:15 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85189 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 0F3CBA09E9; Tue, 15 Dec 2020 08:32:30 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 9F419C9CA; Tue, 15 Dec 2020 08:31:47 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id DA20EC9AE for ; Tue, 15 Dec 2020 08:31:45 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:41 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkG005436; Tue, 15 Dec 2020 09:31:41 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com Date: Tue, 15 Dec 2020 09:31:15 +0200 Message-Id: <20201215073119.404947-5-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 4/8] common/mlx5: add DevX API to create ASO flow meter object X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add DevX API to create ASO flow meter object. Signed-off-by: Li Zhang --- drivers/common/mlx5/mlx5_devx_cmds.c | 54 ++++++++++++++++++++++++++++ drivers/common/mlx5/mlx5_devx_cmds.h | 18 +++++++++- drivers/common/mlx5/version.map | 3 +- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index 765553ed6b..742c6a4997 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -2068,6 +2068,60 @@ mlx5_devx_cmd_create_flow_hit_aso_obj(void *ctx, uint32_t pd) return flow_hit_aso_obj; } +/** + * Create general object of type FLOW_METER_ASO using DevX API. + * + * @param[in] ctx + * Context returned from mlx5 open_device() glue function. + * @param [in] pd + * PD value to associate the FLOW_METER_ASO object with. + * @param [in] log_obj_size + * log_obj_size define to allocate number of 2 * meters + * in one FLOW_METER_ASO object. + * + * @return + * The DevX object created, NULL otherwise and rte_errno is set. + */ +struct mlx5_devx_obj * +mlx5_devx_cmd_create_flow_meter_aso_obj(void *ctx, uint32_t pd, + uint32_t log_obj_size) +{ + uint32_t in[MLX5_ST_SZ_DW(create_flow_meter_aso_in)] = {0}; + uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + struct mlx5_devx_obj *flow_meter_aso_obj; + void *ptr; + + flow_meter_aso_obj = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(*flow_meter_aso_obj), + 0, SOCKET_ID_ANY); + if (!flow_meter_aso_obj) { + DRV_LOG(ERR, "Failed to allocate FLOW_METER_ASO object data"); + rte_errno = ENOMEM; + return NULL; + } + ptr = MLX5_ADDR_OF(create_flow_meter_aso_in, in, hdr); + MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type, + MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO); + MLX5_SET(general_obj_in_cmd_hdr, ptr, log_obj_range, + log_obj_size); + ptr = MLX5_ADDR_OF(create_flow_meter_aso_in, in, flow_meter_aso); + MLX5_SET(flow_meter_aso, ptr, access_pd, pd); + flow_meter_aso_obj->obj = mlx5_glue->devx_obj_create( + ctx, in, sizeof(in), + out, sizeof(out)); + if (!flow_meter_aso_obj->obj) { + rte_errno = errno; + DRV_LOG(ERR, "Failed to create FLOW_METER_ASO obj using DevX."); + mlx5_free(flow_meter_aso_obj); + return NULL; + } + flow_meter_aso_obj->id = MLX5_GET(general_obj_out_cmd_hdr, + out, obj_id); + return flow_meter_aso_obj; +} + /** * Create general object of type GENEVE TLV option using DevX API. * diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h index 13338c0e39..dbd7b66257 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.h +++ b/drivers/common/mlx5/mlx5_devx_cmds.h @@ -517,5 +517,21 @@ int mlx5_devx_cmd_query_virtio_q_counters(struct mlx5_devx_obj *couners_obj, __rte_internal struct mlx5_devx_obj *mlx5_devx_cmd_create_flow_hit_aso_obj(void *ctx, uint32_t pd); - +/** + * Create general object of type FLOW_METER_ASO using DevX API.. + * + * @param[in] ctx + * Device context. + * @param [in] pd + * PD value to associate the FLOW_METER_ASO object with. + * @param [in] log_obj_size + * log_obj_size define to allocate number of 2 * meters + * in one FLOW_METER_ASO object. + * + * @return + * The DevX object created, NULL otherwise and rte_errno is set. + */ +__rte_internal +struct mlx5_devx_obj *mlx5_devx_cmd_create_flow_meter_aso_obj(void *ctx, + uint32_t pd, uint32_t log_obj_size); #endif /* RTE_PMD_MLX5_DEVX_CMDS_H_ */ diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map index 3c403d2feb..51b6fd7326 100644 --- a/drivers/common/mlx5/version.map +++ b/drivers/common/mlx5/version.map @@ -21,7 +21,8 @@ INTERNAL { mlx5_devx_cmd_create_tis; mlx5_devx_cmd_create_virtio_q_counters; mlx5_devx_cmd_create_virtq; - mlx5_devx_cmd_create_flow_hit_aso_obj; + mlx5_devx_cmd_create_flow_hit_aso_obj; + mlx5_devx_cmd_create_flow_meter_aso_obj; mlx5_devx_cmd_create_geneve_tlv_option; mlx5_devx_cmd_destroy; mlx5_devx_cmd_flow_counter_alloc; From patchwork Tue Dec 15 07:31:16 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85191 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id C5C74A09E9; Tue, 15 Dec 2020 08:33:09 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 342D3C9EA; Tue, 15 Dec 2020 08:31:51 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id EEA11C9CE for ; Tue, 15 Dec 2020 08:31:45 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:41 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkH005436; Tue, 15 Dec 2020 09:31:41 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com Date: Tue, 15 Dec 2020 09:31:16 +0200 Message-Id: <20201215073119.404947-6-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 5/8] net/mlx5: flow meter pool to manage meter object X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add ASO flow meter pool to manage meter object Signed-off-by: Li Zhang --- drivers/net/mlx5/mlx5.c | 2 +- drivers/net/mlx5/mlx5.h | 190 ++++++++++++- drivers/net/mlx5/mlx5_flow.c | 44 ++- drivers/net/mlx5/mlx5_flow.h | 186 ++++--------- drivers/net/mlx5/mlx5_flow_dv.c | 208 +++++++++++++- drivers/net/mlx5/mlx5_flow_meter.c | 426 +++++++++++++++++------------ 6 files changed, 723 insertions(+), 333 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 71a81e7f7b..a66e441f67 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -280,7 +280,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = { }, #endif [MLX5_IPOOL_MTR] = { - .size = sizeof(struct mlx5_flow_meter), + .size = sizeof(struct mlx5_legacy_flow_meter), .trunk_size = 64, .grow_trunk = 3, .grow_shift = 2, diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 9e59bfb375..41f2aed53b 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -597,6 +598,177 @@ struct mlx5_dev_shared_port { /* Aging information for per port. */ }; +/*ASO flow meter structures*/ +/* Modify this value if enum rte_mtr_color changes. */ +#define RTE_MTR_DROPPED RTE_COLORS + +/* Meter policer statistics */ +struct mlx5_flow_policer_stats { + uint32_t cnt[RTE_COLORS + 1]; + /**< Color counter, extra for drop. */ + uint64_t stats_mask; + /**< Statistics mask for the colors. */ +}; + +/* Meter table structure. */ +struct mlx5_meter_domain_info { + struct mlx5_flow_tbl_resource *tbl; + /**< Meter table. */ + struct mlx5_flow_tbl_resource *sfx_tbl; + /**< Meter suffix table. */ + void *any_matcher; + /**< Meter color not match default criteria. */ + void *color_matcher; + /**< Meter color match criteria. */ + void *jump_actn; + /**< Meter match action. */ + void *policer_rules[RTE_MTR_DROPPED + 1]; + /**< Meter policer for the match. */ +}; + +/* Meter table set for TX RX FDB. */ +struct mlx5_meter_domains_infos { + uint32_t ref_cnt; + /**< Table user count. */ + struct mlx5_meter_domain_info egress; + /**< TX meter table. */ + struct mlx5_meter_domain_info ingress; + /**< RX meter table. */ + struct mlx5_meter_domain_info transfer; + /**< FDB meter table. */ + void *drop_actn; + /**< Drop action as not matched. */ + void *count_actns[RTE_MTR_DROPPED + 1]; + /**< Counters for match and unmatched statistics. */ + void *meter_action; + /**< Flow meter action. */ +}; + +/* Meter parameter structure. */ +struct mlx5_flow_meter_info { + uint32_t meter_id; + /**< Meter id. */ + struct mlx5_flow_meter_profile *profile; + /**< Meter profile parameters. */ + rte_spinlock_t sl; /**< Meter action spinlock. */ + /** Policer actions (per meter output color). */ + enum rte_mtr_policer_action action[RTE_COLORS]; + /** Set of stats counters to be enabled. + * @see enum rte_mtr_stats_type + */ + uint64_t stats_mask; + /**< Rule applies to ingress traffic. */ + uint32_t ingress:1; + /**< Rule applies to egress traffic. */ + uint32_t egress:1; + /** + * Instead of simply matching the properties of traffic as it would + * appear on a given DPDK port ID, enabling this attribute transfers + * a flow rule to the lowest possible level of any device endpoints + * found in the pattern. + * + * When supported, this effectively enables an application to + * re-route traffic not necessarily intended for it (e.g. coming + * from or addressed to different physical ports, VFs or + * applications) at the device level. + * + * It complements the behavior of some pattern items such as + * RTE_FLOW_ITEM_TYPE_PHY_PORT and is meaningless without them. + * + * When transferring flow rules, ingress and egress attributes keep + * their original meaning, as if processing traffic emitted or + * received by the application. + */ + uint32_t transfer:1; + struct mlx5_meter_domains_infos *mfts; + /**< Flow table created for this meter. */ + struct mlx5_flow_policer_stats policer_stats; + /**< Meter policer statistics. */ + uint32_t ref_cnt; + /**< Use count. */ + uint32_t active_state:1; + /**< Meter hw active state. */ + uint32_t shared:1; + /**< Meter shared or not. */ + uint32_t is_enable:1; + /**< Meter disable/enable state. */ +}; + +/* RFC2697 parameter structure. */ +struct mlx5_flow_meter_srtcm_rfc2697_prm { + rte_be32_t cbs_cir; + /* + * bit 24-28: cbs_exponent, bit 16-23 cbs_mantissa, + * bit 8-12: cir_exponent, bit 0-7 cir_mantissa. + */ + rte_be32_t ebs_eir; + /* + * bit 24-28: ebs_exponent, bit 16-23 ebs_mantissa, + * bit 8-12: eir_exponent, bit 0-7 eir_mantissa. + */ +}; + +/* Flow meter profile structure. */ +struct mlx5_flow_meter_profile { + TAILQ_ENTRY(mlx5_flow_meter_profile) next; + /**< Pointer to the next flow meter structure. */ + uint32_t id; /**< Profile id. */ + struct rte_mtr_meter_profile profile; /**< Profile detail. */ + union { + struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm_prm; + /**< srtcm_rfc2697 struct. */ + }; + uint32_t ref_cnt; /**< Use count. */ +}; + +/* 2 meters in each ASO cache line */ +#define MLX5_MTRS_CONTAINER_RESIZE 64 +/* + * The pool index and offset of meter in the pool array makes up the + * meter index. In case the meter is from pool 0 and offset 0, it + * should plus 1 to avoid index 0, since 0 means invalid meter index + * currently. + */ +#define MLX5_MAKE_MTR_IDX(pi, offset) \ + ((pi) * MLX5_ASO_MTRS_PER_POOL + (offset) + 1) + +/*aso flow meter state*/ +enum mlx5_aso_mtr_state { + ASO_METER_FREE, /* In free list. */ + ASO_METER_WAIT, /* ACCESS_ASO WQE in progress. */ + ASO_METER_READY, /* CQE received. */ +}; + +/* Generic aso_flow_meter information. */ +struct mlx5_aso_mtr { + LIST_ENTRY(mlx5_aso_mtr) next; + struct mlx5_flow_meter_info fm; + /**< Pointer to the next aso flow meter structure. */ + uint8_t state; /**< ASO flow meter state. */ + uint8_t offset; +}; + +/* Generic aso_flow_meter pool structure. */ +struct mlx5_aso_mtr_pool { + struct mlx5_aso_mtr mtrs[MLX5_ASO_MTRS_PER_POOL]; + /*Must be the first in pool*/ + struct mlx5_devx_obj *devx_obj; + /* The devx object of the minimum aso flow meter ID. */ + uint32_t index; /* Pool index in management structure. */ +}; + +LIST_HEAD(aso_meter_list, mlx5_aso_mtr); +/* Pools management structure for ASO flow meter pools. */ +struct mlx5_aso_mtr_pools_mng { + volatile uint16_t n_valid; /* Number of valid pools. */ + uint16_t n; /* Number of pools. */ + rte_spinlock_t mtrsl; /* The ASO flow meter free list lock. */ + struct mlx5_l3t_tbl *mtr_idx_tbl; /* Meter index lookup table. */ + struct aso_meter_list meters; /* Free ASO flow meter list. */ + struct mlx5_aso_sq sq; /*SQ using by ASO flow meter. */ + struct mlx5_aso_mtr_pool **pools; /* ASO flow meter pool array. */ +}; + /* Table key of the hash organization. */ union mlx5_flow_tbl_key { struct { @@ -718,6 +890,7 @@ struct mlx5_dev_ctx_shared { uint16_t bond_dev; /* Bond primary device id. */ uint32_t devx:1; /* Opened with DV. */ uint32_t flow_hit_aso_en:1; /* Flow Hit ASO is supported. */ + uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */ uint32_t eqn; /* Event Queue number. */ uint32_t max_port; /* Maximal IB device port index. */ void *ctx; /* Verbs/DV/DevX context. */ @@ -778,6 +951,8 @@ struct mlx5_dev_ctx_shared { struct mlx5_geneve_tlv_option_resource *geneve_tlv_option_resource; /* Management structure for geneve tlv option */ rte_spinlock_t geneve_tlv_opt_sl; /* Lock for geneve tlv resource */ + struct mlx5_aso_mtr_pools_mng *mtrmng; + /* Meter pools management structure. */ struct mlx5_dev_shared_port port[]; /* per device port data array. */ }; @@ -792,7 +967,7 @@ struct mlx5_proc_priv { /* MTR profile list. */ TAILQ_HEAD(mlx5_mtr_profiles, mlx5_flow_meter_profile); /* MTR list. */ -TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter); +TAILQ_HEAD(mlx5_legacy_flow_meters, mlx5_legacy_flow_meter); /* RSS description. */ struct mlx5_flow_rss_desc { @@ -1013,7 +1188,7 @@ struct mlx5_priv { uint8_t mtr_sfx_reg; /* Meter prefix-suffix flow match REG_C. */ uint8_t mtr_color_reg; /* Meter color match REG_C. */ struct mlx5_mtr_profiles flow_meter_profiles; /* MTR profile list. */ - struct mlx5_flow_meters flow_meters; /* MTR list. */ + struct mlx5_legacy_flow_meters flow_meters; /* MTR list. */ uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */ uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */ struct mlx5_mp_id mp_id; /* ID of a multi-process process */ @@ -1275,14 +1450,15 @@ int mlx5_pmd_socket_init(void); /* mlx5_flow_meter.c */ int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); -struct mlx5_flow_meter *mlx5_flow_meter_find(struct mlx5_priv *priv, - uint32_t meter_id); -struct mlx5_flow_meter *mlx5_flow_meter_attach +struct mlx5_flow_meter_info *mlx5_flow_meter_find(struct mlx5_priv *priv, + uint32_t meter_id, uint32_t *mtr_idx); +struct mlx5_flow_meter_info *mlx5_flow_meter_attach (struct mlx5_priv *priv, uint32_t meter_id, const struct rte_flow_attr *attr, + uint32_t *mtr_idx, struct rte_flow_error *error); -void mlx5_flow_meter_detach(struct mlx5_flow_meter *fm); +void mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm); /* mlx5_os.c */ struct rte_pci_driver; @@ -1327,7 +1503,7 @@ void mlx5_txpp_interrupt_handler(void *cb_arg); eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev); -/* mlx5_flow_age.c */ +/* mlx5_flow_aso.c */ int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh); int mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh); diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 1cfa817ae9..2f047bbe01 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -6467,7 +6467,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, */ struct mlx5_meter_domains_infos * mlx5_flow_create_mtr_tbls(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) + const struct mlx5_flow_meter_info *fm) { const struct mlx5_flow_driver_ops *fops; @@ -6511,7 +6511,7 @@ mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, */ int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; @@ -6533,7 +6533,7 @@ mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, */ int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { const struct mlx5_flow_driver_ops *fops; @@ -6542,6 +6542,44 @@ mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, return fops->destroy_policer_rules(dev, fm, attr); } +/** + * Allcate the needed aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * + * @return + * Index to aso flow meter on success, NULL otherwise. + */ +uint32_t +mlx5_flow_mtr_alloc(struct rte_eth_dev *dev) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + return fops->create_meter(dev); +} + +/** + * Free the aso flow meter id. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] mtr_idx + * Index to aso flow meter to be free. + * + * @return + * 0 on success. + */ +void +mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + const struct mlx5_flow_driver_ops *fops; + + fops = flow_get_drv_ops(MLX5_FLOW_TYPE_DV); + fops->free_meter(dev, mtr_idx); +} + /** * Allocate a counter. * diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 6bb7bae1bf..1c01c611e4 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -811,136 +811,17 @@ struct mlx5_flow { #define MLX5_FLOW_METER_DISABLE 0 #define MLX5_FLOW_METER_ENABLE 1 -#define MLX5_MAN_WIDTH 8 -/* Modify this value if enum rte_mtr_color changes. */ -#define RTE_MTR_DROPPED RTE_COLORS - -/* Meter policer statistics */ -struct mlx5_flow_policer_stats { - uint32_t cnt[RTE_COLORS + 1]; - /**< Color counter, extra for drop. */ - uint64_t stats_mask; - /**< Statistics mask for the colors. */ -}; - -/* Meter table structure. */ -struct mlx5_meter_domain_info { - struct mlx5_flow_tbl_resource *tbl; - /**< Meter table. */ - struct mlx5_flow_tbl_resource *sfx_tbl; - /**< Meter suffix table. */ - void *any_matcher; - /**< Meter color not match default criteria. */ - void *color_matcher; - /**< Meter color match criteria. */ - void *jump_actn; - /**< Meter match action. */ - void *policer_rules[RTE_MTR_DROPPED + 1]; - /**< Meter policer for the match. */ -}; - -/* Meter table set for TX RX FDB. */ -struct mlx5_meter_domains_infos { - uint32_t ref_cnt; - /**< Table user count. */ - struct mlx5_meter_domain_info egress; - /**< TX meter table. */ - struct mlx5_meter_domain_info ingress; - /**< RX meter table. */ - struct mlx5_meter_domain_info transfer; - /**< FDB meter table. */ - void *drop_actn; - /**< Drop action as not matched. */ - void *count_actns[RTE_MTR_DROPPED + 1]; - /**< Counters for match and unmatched statistics. */ - uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; - /**< Flow meter parameter. */ - size_t fmp_size; - /**< Flow meter parameter size. */ - void *meter_action; - /**< Flow meter action. */ -}; +#define MLX5_ASO_CQE_RESPONSE_DELAY 10 +#define MLX5_MTR_POLL_CQE_TIMES 100000u /* 1s*/ -/* Meter parameter structure. */ -struct mlx5_flow_meter { - TAILQ_ENTRY(mlx5_flow_meter) next; +#define MLX5_MAN_WIDTH 8 +/* Legacy Meter parameter structure. */ +struct mlx5_legacy_flow_meter { + struct mlx5_flow_meter_info fm; + /* Must be the first in struct. */ + TAILQ_ENTRY(mlx5_legacy_flow_meter) next; /**< Pointer to the next flow meter structure. */ uint32_t idx; /* Index to meter object. */ - uint32_t meter_id; - /**< Meter id. */ - struct mlx5_flow_meter_profile *profile; - /**< Meter profile parameters. */ - - rte_spinlock_t sl; /**< Meter action spinlock. */ - - /** Policer actions (per meter output color). */ - enum rte_mtr_policer_action action[RTE_COLORS]; - - /** Set of stats counters to be enabled. - * @see enum rte_mtr_stats_type - */ - uint64_t stats_mask; - - /**< Rule applies to ingress traffic. */ - uint32_t ingress:1; - - /**< Rule applies to egress traffic. */ - uint32_t egress:1; - /** - * Instead of simply matching the properties of traffic as it would - * appear on a given DPDK port ID, enabling this attribute transfers - * a flow rule to the lowest possible level of any device endpoints - * found in the pattern. - * - * When supported, this effectively enables an application to - * re-route traffic not necessarily intended for it (e.g. coming - * from or addressed to different physical ports, VFs or - * applications) at the device level. - * - * It complements the behavior of some pattern items such as - * RTE_FLOW_ITEM_TYPE_PHY_PORT and is meaningless without them. - * - * When transferring flow rules, ingress and egress attributes keep - * their original meaning, as if processing traffic emitted or - * received by the application. - */ - uint32_t transfer:1; - struct mlx5_meter_domains_infos *mfts; - /**< Flow table created for this meter. */ - struct mlx5_flow_policer_stats policer_stats; - /**< Meter policer statistics. */ - uint32_t ref_cnt; - /**< Use count. */ - uint32_t active_state:1; - /**< Meter state. */ - uint32_t shared:1; - /**< Meter shared or not. */ -}; - -/* RFC2697 parameter structure. */ -struct mlx5_flow_meter_srtcm_rfc2697_prm { - /* green_saturation_value = cbs_mantissa * 2^cbs_exponent */ - uint32_t cbs_exponent:5; - uint32_t cbs_mantissa:8; - /* cir = 8G * cir_mantissa * 1/(2^cir_exponent) Bytes/Sec */ - uint32_t cir_exponent:5; - uint32_t cir_mantissa:8; - /* yellow _saturation_value = ebs_mantissa * 2^ebs_exponent */ - uint32_t ebs_exponent:5; - uint32_t ebs_mantissa:8; -}; - -/* Flow meter profile structure. */ -struct mlx5_flow_meter_profile { - TAILQ_ENTRY(mlx5_flow_meter_profile) next; - /**< Pointer to the next flow meter structure. */ - uint32_t meter_profile_id; /**< Profile id. */ - struct rte_mtr_meter_profile profile; /**< Profile detail. */ - union { - struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm_prm; - /**< srtcm_rfc2697 struct. */ - }; - uint32_t ref_cnt; /**< Use count. */ }; #define MLX5_MAX_TUNNELS 256 @@ -1066,7 +947,7 @@ struct rte_flow { /**< Device flow handles that are part of the flow. */ uint32_t drv_type:2; /**< Driver type. */ uint32_t tunnel:1; - uint32_t meter:16; /**< Holds flow meter id. */ + uint32_t meter:24; /**< Holds flow meter id. */ uint32_t rix_mreg_copy; /**< Index to metadata register copy table resource. */ uint32_t counter; /**< Holds flow counter. */ @@ -1135,6 +1016,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ + struct mlx5_flow_meter_info *fm;/* the flow meter pointer. */ }; struct mlx5_flow_split_info { @@ -1175,18 +1057,22 @@ typedef int (*mlx5_flow_query_t)(struct rte_eth_dev *dev, void *data, struct rte_flow_error *error); typedef struct mlx5_meter_domains_infos *(*mlx5_flow_create_mtr_tbls_t) - (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + (struct rte_eth_dev *dev, + const struct mlx5_flow_meter_info *fm); typedef int (*mlx5_flow_destroy_mtr_tbls_t)(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbls); typedef int (*mlx5_flow_create_policer_rules_t) (struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); typedef int (*mlx5_flow_destroy_policer_rules_t) (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm, + const struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); +typedef uint32_t (*mlx5_flow_mtr_alloc_t) + (struct rte_eth_dev *dev); +typedef void (*mlx5_flow_mtr_free_t)(struct rte_eth_dev *dev, + uint32_t mtr_idx); typedef uint32_t (*mlx5_flow_counter_alloc_t) (struct rte_eth_dev *dev); typedef void (*mlx5_flow_counter_free_t)(struct rte_eth_dev *dev, @@ -1241,6 +1127,8 @@ struct mlx5_flow_driver_ops { mlx5_flow_destroy_mtr_tbls_t destroy_mtr_tbls; mlx5_flow_create_policer_rules_t create_policer_rules; mlx5_flow_destroy_policer_rules_t destroy_policer_rules; + mlx5_flow_mtr_alloc_t create_meter; + mlx5_flow_mtr_free_t free_meter; mlx5_flow_counter_alloc_t counter_alloc; mlx5_flow_counter_free_t counter_free; mlx5_flow_counter_query_t counter_query; @@ -1300,6 +1188,32 @@ tunnel_use_standard_attr_group_translate return verdict; } +/** + * Get DV flow aso meter by index. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] idx + * mlx5 flow aso meter index in the container. + * @param[out] ppool + * mlx5 flow aso meter pool in the container, + * + * @return + * Pointer to the aso meter, NULL otherwise. + */ +static inline struct mlx5_aso_mtr * +mlx5_aso_meter_by_idx(struct mlx5_priv *priv, uint32_t idx) +{ + struct mlx5_aso_mtr_pool *pool; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + + /* Decrease to original index. */ + idx--; + MLX5_ASSERT(idx / MLX5_ASO_MTRS_PER_POOL < mtrmng->n); + pool = mtrmng->pools[idx / MLX5_ASO_MTRS_PER_POOL]; + return &pool->mtrs[idx % MLX5_ASO_MTRS_PER_POOL]; +} + int mlx5_flow_group_to_table(struct rte_eth_dev *dev, const struct mlx5_flow_tunnel *tunnel, uint32_t group, uint32_t *table, @@ -1434,14 +1348,14 @@ int mlx5_flow_validate_item_ecpri(const struct rte_flow_item *item, struct rte_flow_error *error); struct mlx5_meter_domains_infos *mlx5_flow_create_mtr_tbls (struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm); + const struct mlx5_flow_meter_info *fm); int mlx5_flow_destroy_mtr_tbls(struct rte_eth_dev *dev, struct mlx5_meter_domains_infos *tbl); int mlx5_flow_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr); int mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error); @@ -1537,4 +1451,6 @@ struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev, int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev, const struct rte_flow_item *item, struct rte_flow_error *error); +uint32_t mlx5_flow_mtr_alloc(struct rte_eth_dev *dev); +void mlx5_flow_mtr_free(struct rte_eth_dev *dev, uint32_t mtr_idx); #endif /* RTE_PMD_MLX5_FLOW_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 7905923665..5e1eb51c43 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -4188,7 +4188,7 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, { struct mlx5_priv *priv = dev->data->dev_private; const struct rte_flow_action_meter *am = action->conf; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; if (!am) return rte_flow_error_set(error, EINVAL, @@ -4208,7 +4208,7 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "meter action not supported"); - fm = mlx5_flow_meter_find(priv, am->mtr_id); + fm = mlx5_flow_meter_find(priv, am->mtr_id, NULL); if (!fm) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, @@ -5236,6 +5236,184 @@ flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t counter) } } +/** + * Find meter by index. + * + * @param priv + * Pointer to mlx5_priv. + * @param idx + * Meter index. + * + * @return + * Pointer to the profile found on success, NULL otherwise. + */ +static struct mlx5_flow_meter_info * +flow_dv_meter_find_by_idx(struct mlx5_priv *priv, uint32_t idx) +{ + struct mlx5_aso_mtr *aso_mtr; + + if (priv->sh->meter_aso_en) { + aso_mtr = mlx5_aso_meter_by_idx(priv, idx); + return &aso_mtr->fm; + } else { + return mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], idx); + } +} + +/** + * Resize a meter id container. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * 0 on success, otherwise negative errno value and rte_errno is set. + */ +static int +flow_dv_mtr_container_resize(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + void *old_pools = mtrmng->pools; + uint32_t resize = mtrmng->n + MLX5_MTRS_CONTAINER_RESIZE; + uint32_t mem_size = sizeof(struct mlx5_aso_mtr_pool *) * resize; + void *pools = mlx5_malloc(MLX5_MEM_ZERO, mem_size, 0, SOCKET_ID_ANY); + + if (!pools) { + rte_errno = ENOMEM; + return -ENOMEM; + } + if (old_pools) + memcpy(pools, old_pools, mtrmng->n * + sizeof(struct mlx5_aso_mtr_pool *)); + mtrmng->n = resize; + mtrmng->pools = pools; + if (old_pools) + mlx5_free(old_pools); + return 0; +} + +/** + * Prepare a new meter and/or a new meter pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[out] mtr_free + * Where to put the pointer of a new meter.g. + * + * @return + * The meter pool pointer and @mtr_free is set on success, + * NULL otherwise and rte_errno is set. + */ +static struct mlx5_aso_mtr_pool * +flow_dv_mtr_pool_create(struct rte_eth_dev *dev, + struct mlx5_aso_mtr **mtr_free) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr_pool *pool = NULL; + struct mlx5_devx_obj *dcs = NULL; + uint32_t i; + uint32_t log_obj_size; + + log_obj_size = rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); + dcs = mlx5_devx_cmd_create_flow_meter_aso_obj(priv->sh->ctx, + priv->sh->pdn, log_obj_size); + if (!dcs) { + rte_errno = ENODATA; + return NULL; + } + pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY); + if (!pool) { + rte_errno = ENOMEM; + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + pool->devx_obj = dcs; + pool->index = mtrmng->n_valid; + if (pool->index == mtrmng->n && flow_dv_mtr_container_resize(dev)) { + mlx5_free(pool); + claim_zero(mlx5_devx_cmd_destroy(dcs)); + return NULL; + } + mtrmng->pools[pool->index] = pool; + mtrmng->n_valid++; + for (i = 1; i < MLX5_ASO_MTRS_PER_POOL; ++i) { + pool->mtrs[i].offset = i; + pool->mtrs[i].fm.meter_id = UINT32_MAX; + LIST_INSERT_HEAD(&mtrmng->meters, + &pool->mtrs[i], next); + } + pool->mtrs[0].offset = 0; + pool->mtrs[0].fm.meter_id = UINT32_MAX; + *mtr_free = &pool->mtrs[0]; + return pool; +} + +/** + * Release a flow meter into pool. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * @param[in] mtr_idx + * Index to aso flow meter. + */ +static void +flow_dv_aso_mtr_release_to_pool(struct rte_eth_dev *dev, uint32_t mtr_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr *aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); + + MLX5_ASSERT(aso_mtr); + rte_spinlock_lock(&mtrmng->mtrsl); + aso_mtr->state = ASO_METER_FREE; + aso_mtr->fm.meter_id = UINT32_MAX; + LIST_INSERT_HEAD(&mtrmng->meters, aso_mtr, next); + rte_spinlock_unlock(&mtrmng->mtrsl); +} + +/** + * Allocate a aso flow meter. + * + * @param[in] dev + * Pointer to the Ethernet device structure. + * + * @return + * Index to aso flow meter on success, 0 otherwise and rte_errno is set. + */ +static uint32_t +flow_dv_mtr_alloc(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_aso_mtr *mtr_free = NULL; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_aso_mtr_pool *pool; + uint32_t mtr_idx = 0; + + if (!priv->config.devx) { + rte_errno = ENOTSUP; + return 0; + } + /* Allocate the flow meter memory. */ + /* Get free meters from management. */ + rte_spinlock_lock(&mtrmng->mtrsl); + mtr_free = LIST_FIRST(&mtrmng->meters); + if (mtr_free) + LIST_REMOVE(mtr_free, next); + if (!mtr_free && !flow_dv_mtr_pool_create(dev, &mtr_free)) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return 0; + } + mtr_free->state = ASO_METER_WAIT; + rte_spinlock_unlock(&mtrmng->mtrsl); + pool = container_of(mtr_free, + struct mlx5_aso_mtr_pool, + mtrs[mtr_free->offset]); + mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset); + return mtr_idx; +} + /** * Verify the @p attributes will be correctly understood by the NIC and store * them in the @p flow if everything is correct. @@ -10132,8 +10310,9 @@ flow_dv_translate(struct rte_eth_dev *dev, struct mlx5_flow_dv_port_id_action_resource port_id_resource; int action_type = actions->type; const struct rte_flow_action *found_action = NULL; - struct mlx5_flow_meter *fm = NULL; + struct mlx5_flow_meter_info *fm = NULL; uint32_t jump_group = 0; + uint32_t mtr_idx; if (!mlx5_flow_os_action_supported(action_type)) return rte_flow_error_set(error, ENOTSUP, @@ -10560,7 +10739,7 @@ flow_dv_translate(struct rte_eth_dev *dev, mtr = actions->conf; if (!flow->meter) { fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, error); + attr, &mtr_idx, error); if (!fm) return rte_flow_error_set(error, rte_errno, @@ -10568,12 +10747,12 @@ flow_dv_translate(struct rte_eth_dev *dev, NULL, "meter not found " "or invalid parameters"); - flow->meter = fm->idx; + flow->meter = mtr_idx; + wks->fm = fm; } /* Set the meter action. */ if (!fm) { - fm = mlx5_ipool_get(priv->sh->ipool - [MLX5_IPOOL_MTR], flow->meter); + fm = wks->fm; if (!fm) return rte_flow_error_set(error, rte_errno, @@ -11699,10 +11878,9 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) flow->counter = 0; } if (flow->meter) { - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; - fm = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_MTR], - flow->meter); + fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (fm) mlx5_flow_meter_detach(fm); flow->meter = 0; @@ -12585,7 +12763,7 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, */ static struct mlx5_meter_domains_infos * flow_dv_create_mtr_tbl(struct rte_eth_dev *dev, - const struct mlx5_flow_meter *fm) + const struct mlx5_flow_meter_info *fm) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_meter_domains_infos *mtb; @@ -12682,7 +12860,7 @@ flow_dv_destroy_domain_policer_rule(struct mlx5_meter_domain_info *dt) */ static int flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, - const struct mlx5_flow_meter *fm, + const struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { struct mlx5_meter_domains_infos *mtb = fm ? fm->mfts : NULL; @@ -12712,7 +12890,7 @@ flow_dv_destroy_policer_rules(struct rte_eth_dev *dev __rte_unused, * 0 on success, -1 otherwise. */ static int -flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, +flow_dv_create_policer_forward_rule(struct mlx5_flow_meter_info *fm, struct mlx5_meter_domain_info *dtb, uint8_t mtr_reg_c) { @@ -12775,7 +12953,7 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter *fm, */ static int flow_dv_create_policer_rules(struct rte_eth_dev *dev, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct rte_flow_attr *attr) { struct mlx5_priv *priv = dev->data->dev_private; @@ -13090,6 +13268,8 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = { .query = flow_dv_query, .create_mtr_tbls = flow_dv_create_mtr_tbl, .destroy_mtr_tbls = flow_dv_destroy_mtr_tbl, + .create_meter = flow_dv_mtr_alloc, + .free_meter = flow_dv_aso_mtr_release_to_pool, .create_policer_rules = flow_dv_create_policer_rules, .destroy_policer_rules = flow_dv_destroy_policer_rules, .counter_alloc = flow_dv_counter_allocate, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 03a5e79eb8..b17a71033d 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -28,40 +28,43 @@ */ static void * mlx5_flow_meter_action_create(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm) + struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER struct mlx5dv_dr_flow_meter_attr mtr_init; - void *attr = fm->mfts->fmp; + uint32_t fmp[MLX5_ST_SZ_DW(flow_meter_parameters)]; struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fm->profile->srtcm_prm; + uint32_t cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + uint32_t ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); + uint32_t val; - fm->mfts->fmp_size = MLX5_ST_SZ_BYTES(flow_meter_parameters); - memset(attr, 0, fm->mfts->fmp_size); - MLX5_SET(flow_meter_parameters, attr, valid, 1); - MLX5_SET(flow_meter_parameters, attr, bucket_overflow, 1); - MLX5_SET(flow_meter_parameters, attr, - start_color, MLX5_FLOW_COLOR_GREEN); - MLX5_SET(flow_meter_parameters, attr, both_buckets_on_green, 0); - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); + memset(fmp, 0, MLX5_ST_SZ_BYTES(flow_meter_parameters)); + MLX5_SET(flow_meter_parameters, fmp, valid, 1); + MLX5_SET(flow_meter_parameters, fmp, bucket_overflow, 1); + MLX5_SET(flow_meter_parameters, fmp, + start_color, MLX5_FLOW_COLOR_GREEN); + MLX5_SET(flow_meter_parameters, fmp, both_buckets_on_green, 0); + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, fmp, cbs_mantissa, val); + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, cir_exponent, val); + val = (cbs_cir & ASO_DSEG_MAN_MASK); + MLX5_SET(flow_meter_parameters, fmp, cir_mantissa, val); + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, fmp, ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, fmp, ebs_mantissa, val); mtr_init.next_table = fm->transfer ? fm->mfts->transfer.tbl->obj : - fm->egress ? fm->mfts->egress.tbl->obj : - fm->mfts->ingress.tbl->obj; + fm->egress ? fm->mfts->egress.tbl->obj : + fm->mfts->ingress.tbl->obj; mtr_init.reg_c_index = priv->mtr_color_reg - REG_C_0; - mtr_init.flow_meter_parameter = fm->mfts->fmp; - mtr_init.flow_meter_parameter_sz = fm->mfts->fmp_size; + mtr_init.flow_meter_parameter = fmp; + mtr_init.flow_meter_parameter_sz = + MLX5_ST_SZ_BYTES(flow_meter_parameters); mtr_init.active = fm->active_state; return mlx5_glue->dv_create_flow_action_meter(&mtr_init); #else @@ -89,7 +92,7 @@ mlx5_flow_meter_profile_find(struct mlx5_priv *priv, uint32_t meter_profile_id) struct mlx5_flow_meter_profile *fmp; TAILQ_FOREACH(fmp, fmps, next) - if (meter_profile_id == fmp->meter_profile_id) + if (meter_profile_id == fmp->id) return fmp; return NULL; } @@ -239,44 +242,51 @@ mlx5_flow_meter_param_fill(struct mlx5_flow_meter_profile *fmp, { struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm = &fmp->srtcm_prm; uint8_t man, exp; + uint32_t cbs_exp, cbs_man, cir_exp, cir_man; + uint32_t ebs_exp, ebs_man; if (fmp->profile.alg != RTE_MTR_SRTCM_RFC2697) return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_METER_PROFILE, NULL, "Metering algorithm not supported."); + /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ + mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir, + &man, &exp); + /* Check if cir mantissa is too large. */ + if (exp > ASO_DSEG_CIR_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, + RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, + "meter profile parameter cir is" + " not supported."); + cir_man = man; + cir_exp = exp; /* cbs = cbs_mantissa * 2^cbs_exponent */ mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.cbs, &man, &exp); - srtcm->cbs_mantissa = man; - srtcm->cbs_exponent = exp; /* Check if cbs mantissa is too large. */ - if (srtcm->cbs_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, + if (exp > ASO_DSEG_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter cbs is" - " invalid."); - /* ebs = ebs_mantissa * 2^ebs_exponent */ + "meter profile parameter cbs is" + " not supported."); + cbs_man = man; + cbs_exp = exp; + srtcm->cbs_cir = rte_cpu_to_be_32(cbs_exp << ASO_DSEG_CBS_EXP_OFFSET | + cbs_man << ASO_DSEG_CBS_MAN_OFFSET | + cir_exp << ASO_DSEG_CIR_EXP_OFFSET | + cir_man); mlx5_flow_meter_xbs_man_exp_calc(fmp->profile.srtcm_rfc2697.ebs, &man, &exp); - srtcm->ebs_mantissa = man; - srtcm->ebs_exponent = exp; /* Check if ebs mantissa is too large. */ - if (srtcm->ebs_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, - RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter ebs is" - " invalid."); - /* cir = 8G * cir_mantissa * 1/(2^cir_exponent)) Bytes/Sec */ - mlx5_flow_meter_cir_man_exp_calc(fmp->profile.srtcm_rfc2697.cir, - &man, &exp); - srtcm->cir_mantissa = man; - srtcm->cir_exponent = exp; - /* Check if cir mantissa is too large. */ - if (srtcm->cir_exponent != exp) - return -rte_mtr_error_set(error, EINVAL, + if (exp > ASO_DSEG_EXP_MASK) + return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_MTR_PARAMS, NULL, - "Metering profile parameter cir is" - " invalid."); + "meter profile parameter ebs is" + " not supported."); + ebs_man = man; + ebs_exp = exp; + srtcm->ebs_eir = rte_cpu_to_be_32(ebs_exp << ASO_DSEG_EBS_EXP_OFFSET | + ebs_man << ASO_DSEG_EBS_MAN_OFFSET); return 0; } @@ -306,7 +316,11 @@ mlx5_flow_mtr_cap_get(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); memset(cap, 0, sizeof(*cap)); - cap->n_max = 1 << qattr->log_max_flow_meter; + if (priv->sh->meter_aso_en) + /* 2 meters per one ASO cache line. */ + cap->n_max = 1 << (qattr->log_max_num_meter_aso + 1); + else + cap->n_max = 1 << qattr->log_max_flow_meter; cap->n_shared_max = cap->n_max; cap->identical = 1; cap->shared_identical = 1; @@ -365,7 +379,7 @@ mlx5_flow_meter_profile_add(struct rte_eth_dev *dev, NULL, "Meter profile memory " "alloc failed."); /* Fill profile info. */ - fmp->meter_profile_id = meter_profile_id; + fmp->id = meter_profile_id; fmp->profile = *profile; /* Fill the flow meter parameters for the PRM. */ ret = mlx5_flow_meter_param_fill(fmp, error); @@ -495,7 +509,7 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, action2error(params->action[i]), NULL, "Recolor action not supported."); /* Validate meter id. */ - if (mlx5_flow_meter_find(priv, meter_id)) + if (mlx5_flow_meter_find(priv, meter_id, NULL)) return -rte_mtr_error_set(error, EEXIST, RTE_MTR_ERROR_TYPE_MTR_ID, NULL, "Meter object already exists."); @@ -520,7 +534,7 @@ mlx5_flow_meter_validate(struct mlx5_priv *priv, uint32_t meter_id, */ static int mlx5_flow_meter_action_modify(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, uint64_t modify_bits, uint32_t active_state) { @@ -529,33 +543,37 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, uint32_t *attr; struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; int ret; + uint32_t cbs_cir, ebs_eir, val; /* Fill command parameters. */ mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; mod_attr.flow_meter_parameter = in; - mod_attr.flow_meter_parameter_sz = fm->mfts->fmp_size; + mod_attr.flow_meter_parameter_sz = + MLX5_ST_SZ_BYTES(flow_meter_parameters); if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) mod_attr.active = !!active_state; else mod_attr.active = 0; attr = in; + cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, cbs_mantissa, val); } if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, cir_exponent, val); + val = cbs_cir & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, cir_mantissa, val); } if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, ebs_mantissa, val); } /* Apply modifications to meter only if it was created. */ if (fm->mfts->meter_action) { @@ -568,26 +586,6 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, /* Update succeedded modify meter parameters. */ if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) fm->active_state = !!active_state; - attr = fm->mfts->fmp; - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - MLX5_SET(flow_meter_parameters, - attr, cbs_exponent, srtcm->cbs_exponent); - MLX5_SET(flow_meter_parameters, - attr, cbs_mantissa, srtcm->cbs_mantissa); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - MLX5_SET(flow_meter_parameters, - attr, cir_exponent, srtcm->cir_exponent); - MLX5_SET(flow_meter_parameters, - attr, cir_mantissa, srtcm->cir_mantissa); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - MLX5_SET(flow_meter_parameters, - attr, ebs_exponent, srtcm->ebs_exponent); - MLX5_SET(flow_meter_parameters, - attr, ebs_mantissa, srtcm->ebs_mantissa); - } - return 0; #else (void)priv; @@ -622,9 +620,10 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, @@ -632,7 +631,8 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, }; int ret; unsigned int i; - uint32_t idx = 0; + struct mlx5_aso_mtr *aso_mtr; + uint32_t mtr_idx; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, @@ -649,12 +649,24 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, NULL, "Meter profile id not valid."); /* Allocate the flow meter memory. */ - fm = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_MTR], &idx); - if (fm == NULL) - return -rte_mtr_error_set(error, ENOMEM, - RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, - "Memory alloc failed for meter."); - fm->idx = idx; + if (priv->sh->meter_aso_en) { + mtr_idx = mlx5_flow_mtr_alloc(dev); + if (!mtr_idx) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Memory alloc failed for meter."); + aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_idx); + fm = &aso_mtr->fm; + } else { + legacy_fm = mlx5_ipool_zmalloc + (priv->sh->ipool[MLX5_IPOOL_MTR], &mtr_idx); + if (legacy_fm == NULL) + return -rte_mtr_error_set(error, ENOMEM, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Memory alloc failed for meter."); + legacy_fm->idx = mtr_idx; + fm = &legacy_fm->fm; + } /* Fill the flow meter parameters. */ fm->meter_id = meter_id; fm->profile = fmp; @@ -674,11 +686,12 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (ret) goto error; /* Add to the flow meter list. */ - TAILQ_INSERT_TAIL(fms, fm, next); + if (!priv->sh->meter_aso_en) + TAILQ_INSERT_TAIL(fms, legacy_fm, next); fm->active_state = 1; /* Config meter starts as active. */ fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; - fm->profile->ref_cnt++; + __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); rte_spinlock_init(&fm->sl); return 0; error: @@ -688,12 +701,56 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) if (fm->policer_stats.cnt[i]) mlx5_counter_free(dev, fm->policer_stats.cnt[i]); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], idx); + if (priv->sh->meter_aso_en) + mlx5_flow_mtr_free(dev, mtr_idx); + else + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], mtr_idx); return -rte_mtr_error_set(error, -ret, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Failed to create devx meter."); } +static int +mlx5_flow_meter_params_flush(struct rte_eth_dev *dev, + struct mlx5_flow_meter_info *fm, + const struct rte_flow_attr *attr, + uint32_t mtr_idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; + struct mlx5_flow_meter_profile *fmp; + struct mlx5_legacy_flow_meter *legacy_fm = NULL; + uint32_t i; + + /* Meter object must not have any owner. */ + MLX5_ASSERT(!fm->ref_cnt); + /* Get meter profile. */ + fmp = fm->profile; + if (fmp == NULL) + return -1; + /* Update dependencies. */ + __atomic_sub_fetch(&fmp->ref_cnt, 1, __ATOMIC_RELAXED); + /* Remove from list. */ + if (!priv->sh->meter_aso_en) { + legacy_fm = container_of(fm, struct mlx5_legacy_flow_meter, fm); + TAILQ_REMOVE(fms, legacy_fm, next); + } + /* Free policer counters. */ + for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) + if (fm->policer_stats.cnt[i]) + mlx5_counter_free(dev, + fm->policer_stats.cnt[i]); + /* Free meter flow table. */ + mlx5_flow_destroy_policer_rules(dev, fm, attr); + mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); + if (priv->sh->meter_aso_en) + mlx5_flow_mtr_free(dev, mtr_idx); + else + mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], + legacy_fm->idx); + return 0; +} + /** * Destroy meter rules. * @@ -712,22 +769,21 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; - struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_flow_meter_info *fm; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; - unsigned int i; + uint32_t mtr_idx = 0; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -737,21 +793,17 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, return -rte_mtr_error_set(error, EBUSY, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter object is being used."); - /* Get the meter profile. */ - fmp = fm->profile; - MLX5_ASSERT(fmp); - /* Update dependencies. */ - fmp->ref_cnt--; - /* Remove from the flow meter list. */ - TAILQ_REMOVE(fms, fm, next); - /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, fm->policer_stats.cnt[i]); - /* Free meter flow table */ - mlx5_flow_destroy_policer_rules(dev, fm, &attr); - mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); + if (priv->sh->meter_aso_en) { + if (mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id)) + return -rte_mtr_error_set(error, EBUSY, + RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, + "Fail to delete ASO Meter in index table."); + } + /* Destroy the meter profile. */ + if (mlx5_flow_meter_params_flush(dev, fm, &attr, mtr_idx)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); return 0; } @@ -772,17 +824,13 @@ mlx5_flow_meter_destroy(struct rte_eth_dev *dev, uint32_t meter_id, */ static int mlx5_flow_meter_modify_state(struct mlx5_priv *priv, - struct mlx5_flow_meter *fm, + struct mlx5_flow_meter_info *fm, uint32_t new_state, struct rte_mtr_error *error) { static const struct mlx5_flow_meter_srtcm_rfc2697_prm srtcm = { - .cbs_exponent = 20, - .cbs_mantissa = 191, - .cir_exponent = 0, - .cir_mantissa = 200, - .ebs_exponent = 0, - .ebs_mantissa = 0, + .cbs_cir = RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL), + .ebs_eir = 0, }; uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; @@ -824,7 +872,7 @@ mlx5_flow_meter_enable(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; int ret; if (!priv->mtr_en) @@ -832,7 +880,7 @@ mlx5_flow_meter_enable(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -865,7 +913,7 @@ mlx5_flow_meter_disable(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; int ret; if (!priv->mtr_en) @@ -873,7 +921,7 @@ mlx5_flow_meter_disable(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -911,7 +959,7 @@ mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flow_meter_profile *fmp; struct mlx5_flow_meter_profile *old_fmp; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; uint64_t modify_bits = MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS | MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR; int ret; @@ -927,7 +975,7 @@ mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, NULL, "Meter profile not found."); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -977,14 +1025,14 @@ mlx5_flow_meter_stats_update(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; if (!priv->mtr_en) return -rte_mtr_error_set(error, ENOTSUP, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1027,7 +1075,7 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, RTE_MTR_STATS_N_PKTS_DROPPED | RTE_MTR_STATS_N_BYTES_DROPPED }; struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; struct mlx5_flow_policer_stats *ps; uint64_t pkts_dropped = 0; uint64_t bytes_dropped = 0; @@ -1041,7 +1089,7 @@ mlx5_flow_meter_stats_read(struct rte_eth_dev *dev, RTE_MTR_ERROR_TYPE_UNSPECIFIED, NULL, "Meter is not supported"); /* Meter object must exist. */ - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, NULL); if (fm == NULL) return -rte_mtr_error_set(error, ENOENT, RTE_MTR_ERROR_TYPE_MTR_ID, @@ -1134,15 +1182,41 @@ mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) * @return * Pointer to the profile found on success, NULL otherwise. */ -struct mlx5_flow_meter * -mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) +struct mlx5_flow_meter_info * +mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, + uint32_t *mtr_idx) { - struct mlx5_flow_meters *fms = &priv->flow_meters; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; + struct mlx5_aso_mtr *aso_mtr; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + union mlx5_l3t_data data; - TAILQ_FOREACH(fm, fms, next) - if (meter_id == fm->meter_id) - return fm; + if (priv->sh->meter_aso_en) { + rte_spinlock_lock(&mtrmng->mtrsl); + if (!mtrmng->n_valid) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return NULL; + } + if (mlx5_l3t_get_entry(mtrmng->mtr_idx_tbl, meter_id, &data) || + !data.dword) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return NULL; + } + if (mtr_idx) + *mtr_idx = data.dword; + aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); + mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id); + if (meter_id == aso_mtr->fm.meter_id) { + rte_spinlock_unlock(&mtrmng->mtrsl); + return &aso_mtr->fm; + } + rte_spinlock_unlock(&mtrmng->mtrsl); + } else { + TAILQ_FOREACH(legacy_fm, fms, next) + if (meter_id == legacy_fm->fm.meter_id) + return &legacy_fm->fm; + } return NULL; } @@ -1162,15 +1236,15 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id) * * @return the flow meter pointer, NULL otherwise. */ -struct mlx5_flow_meter * +struct mlx5_flow_meter_info * mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, - const struct rte_flow_attr *attr, + const struct rte_flow_attr *attr, uint32_t *mtr_idx, struct rte_flow_error *error) { - struct mlx5_flow_meter *fm; + struct mlx5_flow_meter_info *fm; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id); + fm = mlx5_flow_meter_find(priv, meter_id, mtr_idx); if (fm == NULL) { rte_flow_error_set(error, ENOENT, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, @@ -1220,7 +1294,7 @@ mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, * Pointer to flow meter. */ void -mlx5_flow_meter_detach(struct mlx5_flow_meter *fm) +mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER rte_spinlock_lock(&fm->sl); @@ -1253,40 +1327,46 @@ int mlx5_flow_meter_flush(struct rte_eth_dev *dev, struct rte_mtr_error *error) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_flow_meters *fms = &priv->flow_meters; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; + struct mlx5_legacy_flow_meters *fms = &priv->flow_meters; struct mlx5_mtr_profiles *fmps = &priv->flow_meter_profiles; struct mlx5_flow_meter_profile *fmp; - struct mlx5_flow_meter *fm; + struct mlx5_legacy_flow_meter *legacy_fm; + struct mlx5_flow_meter_info *fm; + struct mlx5_aso_mtr_pool *mtr_pool; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, .transfer = priv->config.dv_esw_en ? 1 : 0, }; void *tmp; - uint32_t i; + uint32_t i, offset, mtr_idx; - TAILQ_FOREACH_SAFE(fm, fms, next, tmp) { - /* Meter object must not have any owner. */ - MLX5_ASSERT(!fm->ref_cnt); - /* Get meter profile. */ - fmp = fm->profile; - if (fmp == NULL) - return -rte_mtr_error_set(error, EINVAL, - RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, - NULL, "MTR object meter profile invalid."); - /* Update dependencies. */ - fmp->ref_cnt--; - /* Remove from list. */ - TAILQ_REMOVE(fms, fm, next); - /* Free policer counters. */ - for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) - if (fm->policer_stats.cnt[i]) - mlx5_counter_free(dev, - fm->policer_stats.cnt[i]); - /* Free meter flow table. */ - mlx5_flow_destroy_policer_rules(dev, fm, &attr); - mlx5_flow_destroy_mtr_tbls(dev, fm->mfts); - mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MTR], fm->idx); + if (priv->sh->meter_aso_en) { + i = mtrmng->n_valid; + while (i--) { + mtr_pool = mtrmng->pools[i]; + for (offset = 0; offset < MLX5_ASO_MTRS_PER_POOL; + offset++) { + fm = &mtr_pool->mtrs[offset].fm; + mtr_idx = MLX5_MAKE_MTR_IDX(i, offset); + if (fm->meter_id != UINT32_MAX && + mlx5_flow_meter_params_flush(dev, + fm, &attr, mtr_idx)) + return -rte_mtr_error_set + (error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); + } + } + } else { + TAILQ_FOREACH_SAFE(legacy_fm, fms, next, tmp) { + fm = &legacy_fm->fm; + if (mlx5_flow_meter_params_flush(dev, fm, &attr, 0)) + return -rte_mtr_error_set(error, EINVAL, + RTE_MTR_ERROR_TYPE_METER_PROFILE_ID, + NULL, "MTR object meter profile invalid."); + } } TAILQ_FOREACH_SAFE(fmp, fmps, next, tmp) { /* Check unused. */ From patchwork Tue Dec 15 07:31:17 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85190 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9F0DFA09E9; Tue, 15 Dec 2020 08:32:50 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 6AEB8C9D8; Tue, 15 Dec 2020 08:31:49 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id E20D0C9C8 for ; Tue, 15 Dec 2020 08:31:45 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:41 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkI005436; Tue, 15 Dec 2020 09:31:41 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com Date: Tue, 15 Dec 2020 09:31:17 +0200 Message-Id: <20201215073119.404947-7-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 6/8] net/mlx5: init/uninit flow meter queue for WQE X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Init/uninit flow meter SQ for WQE Signed-off-by: Li Zhang --- drivers/net/mlx5/linux/mlx5_os.c | 17 ++++ drivers/net/mlx5/meson.build | 2 +- drivers/net/mlx5/mlx5.c | 67 ++++++++++++- drivers/net/mlx5/mlx5.h | 20 ++-- drivers/net/mlx5/mlx5_flow.h | 4 +- .../mlx5/{mlx5_flow_age.c => mlx5_flow_aso.c} | 97 ++++++++++++++++--- drivers/net/mlx5/mlx5_flow_dv.c | 7 +- 7 files changed, 187 insertions(+), 27 deletions(-) rename drivers/net/mlx5/{mlx5_flow_age.c => mlx5_flow_aso.c} (87%) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 4c7782f8ef..01b019c28d 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -697,6 +697,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, int own_domain_id = 0; uint16_t port_id; unsigned int i; + uint32_t log_obj_size; #ifdef HAVE_MLX5DV_DR_DEVX_PORT struct mlx5dv_devx_port devx_port = { .comp_mask = 0 }; #endif @@ -1267,6 +1268,22 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, priv->mtr_color_reg); } } + if (config->hca_attr.qos.sup && + config->hca_attr.qos.flow_meter_aso_sup) { + log_obj_size = + rte_log2_u32(MLX5_ASO_MTRS_PER_POOL >> 1); + if (log_obj_size >= + config->hca_attr.qos.log_meter_aso_granularity && + log_obj_size <= + config->hca_attr.qos.log_meter_aso_max_alloc) { + sh->meter_aso_en = 1; + err = mlx5_aso_flow_mtrs_mng_init(sh); + if (err) { + err = -err; + goto error; + } + } + } #endif #ifdef HAVE_MLX5_DR_CREATE_ACTION_ASO if (config->hca_attr.flow_hit_aso && diff --git a/drivers/net/mlx5/meson.build b/drivers/net/mlx5/meson.build index e7495a764a..ad501bd1e2 100644 --- a/drivers/net/mlx5/meson.build +++ b/drivers/net/mlx5/meson.build @@ -16,7 +16,7 @@ sources = files( 'mlx5_flow_meter.c', 'mlx5_flow_dv.c', 'mlx5_flow_verbs.c', - 'mlx5_flow_age.c', + 'mlx5_flow_aso.c', 'mlx5_mac.c', 'mlx5_mr.c', 'mlx5_rss.c', diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index a66e441f67..9d77265e2e 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -383,7 +383,7 @@ mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh) rte_errno = ENOMEM; return -ENOMEM; } - err = mlx5_aso_queue_init(sh); + err = mlx5_aso_queue_init(sh, ASO_OPC_MOD_FLOW_HIT); if (err) { mlx5_free(sh->aso_age_mng); return -1; @@ -405,8 +405,8 @@ mlx5_flow_aso_age_mng_close(struct mlx5_dev_ctx_shared *sh) { int i, j; - mlx5_aso_queue_stop(sh); - mlx5_aso_queue_uninit(sh); + mlx5_aso_flow_hit_queue_poll_stop(sh); + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_FLOW_HIT); if (sh->aso_age_mng->pools) { struct mlx5_aso_age_pool *pool; @@ -544,6 +544,65 @@ mlx5_flow_counters_mng_close(struct mlx5_dev_ctx_shared *sh) memset(&sh->cmng, 0, sizeof(sh->cmng)); } +/** + * Initialize the aso flow meters management structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object to free + */ +int +mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh) +{ + if (sh->mtrmng) + return 0; + sh->mtrmng = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sh->mtrmng), + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); + if (!sh->mtrmng) { + DRV_LOG(ERR, "mlx5_aso_mtr_pools_mng allocation was failed."); + rte_errno = ENOMEM; + return -ENOMEM; + } + sh->mtrmng->mtr_idx_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_DWORD); + if (!sh->mtrmng->mtr_idx_tbl) { + DRV_LOG(ERR, "fail to create meter lookup table."); + rte_errno = ENOMEM; + return -ENOMEM; + } + rte_spinlock_init(&sh->mtrmng->mtrsl); + LIST_INIT(&sh->mtrmng->meters); + return 0; +} + +/** + * Close and release all the resources of + * the ASO flow meter management structure. + * + * @param[in] sh + * Pointer to mlx5_dev_ctx_shared object to free. + */ +static void +mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) +{ + struct mlx5_aso_mtr_pool *mtr_pool; + struct mlx5_aso_mtr_pools_mng *mtrmng = sh->mtrmng; + uint32_t idx; + + mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); + idx = mtrmng->n_valid; + while (idx--) { + mtr_pool = mtrmng->pools[idx]; + claim_zero(mlx5_devx_cmd_destroy + (mtr_pool->devx_obj)); + mtrmng->n_valid--; + mlx5_free(mtr_pool); + } + mlx5_free(sh->mtrmng->pools); + mlx5_l3t_destroy(mtrmng->mtr_idx_tbl); + mtrmng->mtr_idx_tbl = NULL; + mlx5_free(sh->mtrmng); + sh->mtrmng = NULL; +} + /* Send FLOW_AGED event if needed. */ void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh) @@ -1092,6 +1151,8 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh) mlx5_flow_aso_age_mng_close(sh); sh->aso_age_mng = NULL; } + if (sh->mtrmng) + mlx5_aso_flow_mtrs_mng_close(sh); mlx5_flow_ipool_destroy(sh); mlx5_os_dev_shared_handler_uninstall(sh); if (sh->cnt_id_tbl) { diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 41f2aed53b..d0c30939d2 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -500,8 +500,13 @@ struct mlx5_aso_devx_mr { }; struct mlx5_aso_sq_elem { - struct mlx5_aso_age_pool *pool; - uint16_t burst_size; + union { + struct { + struct mlx5_aso_age_pool *pool; + uint16_t burst_size; + }; + struct mlx5_aso_mtr *mtr; + }; }; struct mlx5_aso_sq { @@ -1244,6 +1249,7 @@ int mlx5_hairpin_cap_get(struct rte_eth_dev *dev, bool mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev); int mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev); int mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh); /* mlx5_ethdev.c */ @@ -1505,9 +1511,11 @@ eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev); /* mlx5_flow_aso.c */ -int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh); -int mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh); -int mlx5_aso_queue_stop(struct mlx5_dev_ctx_shared *sh); -void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod); +int mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh); +int mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh); +void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index 1c01c611e4..ab83f67638 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -811,8 +811,8 @@ struct mlx5_flow { #define MLX5_FLOW_METER_DISABLE 0 #define MLX5_FLOW_METER_ENABLE 1 -#define MLX5_ASO_CQE_RESPONSE_DELAY 10 -#define MLX5_MTR_POLL_CQE_TIMES 100000u /* 1s*/ +#define MLX5_ASO_WQE_CQE_RESPONSE_DELAY 10u +#define MLX5_MTR_POLL_WQE_CQE_TIMES 100000u #define MLX5_MAN_WIDTH 8 /* Legacy Meter parameter structure. */ diff --git a/drivers/net/mlx5/mlx5_flow_age.c b/drivers/net/mlx5/mlx5_flow_aso.c similarity index 87% rename from drivers/net/mlx5/mlx5_flow_age.c rename to drivers/net/mlx5/mlx5_flow_aso.c index cea2cf769d..b2555724d2 100644 --- a/drivers/net/mlx5/mlx5_flow_age.c +++ b/drivers/net/mlx5/mlx5_flow_aso.c @@ -196,7 +196,6 @@ mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) } if (sq->cq.cq) mlx5_aso_cq_destroy(&sq->cq); - mlx5_aso_devx_dereg_mr(&sq->mr); memset(sq, 0, sizeof(*sq)); } @@ -207,7 +206,7 @@ mlx5_aso_destroy_sq(struct mlx5_aso_sq *sq) * ASO SQ to initialize. */ static void -mlx5_aso_init_sq(struct mlx5_aso_sq *sq) +mlx5_aso_age_init_sq(struct mlx5_aso_sq *sq) { volatile struct mlx5_aso_wqe *restrict wqe; int i; @@ -233,6 +232,39 @@ mlx5_aso_init_sq(struct mlx5_aso_sq *sq) } } +/** + * Initialize Send Queue used for ASO flow meter access. + * + * @param[in] sq + * ASO SQ to initialize. + */ +static void +mlx5_aso_mtr_init_sq(struct mlx5_aso_sq *sq) +{ + volatile struct mlx5_aso_wqe *restrict wqe; + int i; + int size = 1 << sq->log_desc_n; + uint32_t idx; + + /* All the next fields state should stay constant. */ + for (i = 0, wqe = &sq->wqes[0]; i < size; ++i, ++wqe) { + wqe->general_cseg.sq_ds = rte_cpu_to_be_32((sq->sqn << 8) | + (sizeof(*wqe) >> 4)); + wqe->aso_cseg.operand_masks = RTE_BE32(0u | + (ASO_OPER_LOGICAL_OR << ASO_CSEG_COND_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_1_OPER_OFFSET) | + (ASO_OP_ALWAYS_TRUE << ASO_CSEG_COND_0_OPER_OFFSET) | + (BYTEWISE_64BYTE << ASO_CSEG_DATA_MASK_MODE_OFFSET)); + wqe->general_cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS << + MLX5_COMP_MODE_OFFSET); + for (idx = 0; idx < MLX5_ASO_METERS_PER_WQE; + idx++) + wqe->aso_dseg.mtrs[idx].v_bo_sc_bbog_mm = + RTE_BE32((1 << ASO_DSEG_VALID_OFFSET) | + (MLX5_FLOW_COLOR_GREEN << ASO_DSEG_SC_OFFSET)); + } +} + /** * Create Send Queue used for ASO access. * @@ -267,9 +299,6 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, uint32_t wq_size = sizeof(struct mlx5_aso_wqe) * sq_desc_n; int ret; - if (mlx5_aso_devx_reg_mr(ctx, (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * - sq_desc_n, &sq->mr, socket, pdn)) - return -1; if (mlx5_aso_cq_create(ctx, &sq->cq, log_desc_n, socket, mlx5_os_get_devx_uar_page_id(uar), eqn)) goto error; @@ -326,7 +355,6 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, sq->sqn = sq->sq->id; sq->db_rec = RTE_PTR_ADD(sq->umem_buf, (uintptr_t)(wq_attr->dbr_addr)); sq->uar_addr = (volatile uint64_t *)((uint8_t *)uar->base_addr + 0x800); - mlx5_aso_init_sq(sq); return 0; error: mlx5_aso_destroy_sq(sq); @@ -343,11 +371,37 @@ mlx5_aso_sq_create(void *ctx, struct mlx5_aso_sq *sq, int socket, * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod) { - return mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0, - sh->tx_uar, sh->pdn, sh->eqn, - MLX5_ASO_QUEUE_LOG_DESC); + uint32_t sq_desc_n = 1 << MLX5_ASO_QUEUE_LOG_DESC; + + switch (aso_opc_mod) { + case ASO_OPC_MOD_FLOW_HIT: + if (mlx5_aso_devx_reg_mr(sh->ctx, + (MLX5_ASO_AGE_ACTIONS_PER_POOL / 8) * + sq_desc_n, &sh->aso_age_mng->aso_sq.mr, 0, sh->pdn)) + return -1; + if (mlx5_aso_sq_create(sh->ctx, &sh->aso_age_mng->aso_sq, 0, + sh->tx_uar, sh->pdn, sh->eqn, + MLX5_ASO_QUEUE_LOG_DESC)) { + mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr); + return -1; + } + mlx5_aso_age_init_sq(&sh->aso_age_mng->aso_sq); + break; + case ASO_OPC_MOD_POLICER: + if (mlx5_aso_sq_create(sh->ctx, &sh->mtrmng->sq, 0, + sh->tx_uar, sh->pdn, sh->eqn, + MLX5_ASO_QUEUE_LOG_DESC)) + return -1; + mlx5_aso_mtr_init_sq(&sh->mtrmng->sq); + break; + default: + DRV_LOG(ERR, "Unknown ASO operation mode"); + return -1; + } + return 0; } /** @@ -357,9 +411,24 @@ mlx5_aso_queue_init(struct mlx5_dev_ctx_shared *sh) * Pointer to shared device context. */ void -mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, + enum mlx5_access_aso_opc_mod aso_opc_mod) { - mlx5_aso_destroy_sq(&sh->aso_age_mng->aso_sq); + struct mlx5_aso_sq *sq; + + switch (aso_opc_mod) { + case ASO_OPC_MOD_FLOW_HIT: + mlx5_aso_devx_dereg_mr(&sh->aso_age_mng->aso_sq.mr); + sq = &sh->aso_age_mng->aso_sq; + break; + case ASO_OPC_MOD_POLICER: + sq = &sh->mtrmng->sq; + break; + default: + DRV_LOG(ERR, "Unknown ASO operation mode"); + return; + } + mlx5_aso_destroy_sq(sq); } /** @@ -634,7 +703,7 @@ mlx5_flow_aso_alarm(void *arg) * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh) { if (rte_eal_alarm_set(US_PER_S, mlx5_flow_aso_alarm, sh)) { DRV_LOG(ERR, "Cannot reinitialize ASO age alarm."); @@ -653,7 +722,7 @@ mlx5_aso_queue_start(struct mlx5_dev_ctx_shared *sh) * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -mlx5_aso_queue_stop(struct mlx5_dev_ctx_shared *sh) +mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh) { int retries = 1024; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 5e1eb51c43..9bac7d3a02 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -5283,6 +5283,11 @@ flow_dv_mtr_container_resize(struct rte_eth_dev *dev) rte_errno = ENOMEM; return -ENOMEM; } + if (!mtrmng->n) + if (mlx5_aso_queue_init(priv->sh, ASO_OPC_MOD_POLICER)) { + mlx5_free(pools); + return -ENOMEM; + } if (old_pools) memcpy(pools, old_pools, mtrmng->n * sizeof(struct mlx5_aso_mtr_pool *)); @@ -9980,7 +9985,7 @@ flow_dv_aso_age_pools_resize(struct rte_eth_dev *dev) mlx5_free(old_pools); } else { /* First ASO flow hit allocation - starting ASO data-path. */ - int ret = mlx5_aso_queue_start(priv->sh); + int ret = mlx5_aso_flow_hit_queue_poll_start(priv->sh); if (ret) { mlx5_free(pools); From patchwork Tue Dec 15 07:31:18 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85192 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 5815AA09E9; Tue, 15 Dec 2020 08:33:29 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A16A1C9F4; Tue, 15 Dec 2020 08:31:52 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id E6CD2C9CA for ; Tue, 15 Dec 2020 08:31:45 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:41 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkJ005436; Tue, 15 Dec 2020 09:31:41 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com Date: Tue, 15 Dec 2020 09:31:18 +0200 Message-Id: <20201215073119.404947-8-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 7/8] net/mlx5: aso flow meter send WQE and CQE handle X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" ASO flow meter send WQE and CQE handle functions Signed-off-by: Li Zhang --- drivers/net/mlx5/mlx5.h | 4 + drivers/net/mlx5/mlx5_flow.h | 2 +- drivers/net/mlx5/mlx5_flow_aso.c | 182 +++++++++++++++++++++++++++++ drivers/net/mlx5/mlx5_flow_dv.c | 5 +- drivers/net/mlx5/mlx5_flow_meter.c | 144 ++++++++++++++--------- 5 files changed, 279 insertions(+), 58 deletions(-) diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index d0c30939d2..73d3698f1d 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -1517,5 +1517,9 @@ int mlx5_aso_flow_hit_queue_poll_start(struct mlx5_dev_ctx_shared *sh); int mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh); void mlx5_aso_queue_uninit(struct mlx5_dev_ctx_shared *sh, enum mlx5_access_aso_opc_mod aso_opc_mod); +int mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr); +int mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h index ab83f67638..eaaa923778 100644 --- a/drivers/net/mlx5/mlx5_flow.h +++ b/drivers/net/mlx5/mlx5_flow.h @@ -1016,7 +1016,7 @@ struct mlx5_flow_workspace { struct mlx5_flow_rss_desc rss_desc; uint32_t rssq_num; /* Allocated queue num in rss_desc. */ uint32_t flow_idx; /* Intermediate device flow index. */ - struct mlx5_flow_meter_info *fm;/* the flow meter pointer. */ + uint32_t mtr_idx;/* The aso meter index. */ }; struct mlx5_flow_split_info { diff --git a/drivers/net/mlx5/mlx5_flow_aso.c b/drivers/net/mlx5/mlx5_flow_aso.c index b2555724d2..c9280b1f9d 100644 --- a/drivers/net/mlx5/mlx5_flow_aso.c +++ b/drivers/net/mlx5/mlx5_flow_aso.c @@ -737,3 +737,185 @@ mlx5_aso_flow_hit_queue_poll_stop(struct mlx5_dev_ctx_shared *sh) } return -rte_errno; } + +static uint16_t +mlx5_aso_mtr_sq_enqueue_single(struct mlx5_aso_sq *sq, + struct mlx5_aso_mtr *aso_mtr) +{ + volatile struct mlx5_aso_wqe *wqe = NULL; + struct mlx5_flow_meter_info *fm = NULL; + uint16_t size = 1 << sq->log_desc_n; + uint16_t mask = size - 1; + uint16_t res = size - (uint16_t)(sq->head - sq->tail); + uint32_t dseg_idx = 0; + struct mlx5_aso_mtr_pool *pool = NULL; + + if (unlikely(!res)) { + DRV_LOG(ERR, "Fail: SQ is full and no free WQE to send"); + return 0; + } + wqe = &sq->wqes[sq->head & mask]; + rte_prefetch0(&sq->wqes[(sq->head + 1) & mask]); + /* Fill next WQE. */ + fm = &aso_mtr->fm; + sq->elts[sq->head & mask].mtr = aso_mtr; + pool = container_of(aso_mtr, struct mlx5_aso_mtr_pool, + mtrs[aso_mtr->offset]); + wqe->general_cseg.misc = rte_cpu_to_be_32(pool->devx_obj->id + + (aso_mtr->offset >> 1)); + wqe->general_cseg.opcode = rte_cpu_to_be_32(MLX5_OPCODE_ACCESS_ASO | + (ASO_OPC_MOD_POLICER << + WQE_CSEG_OPC_MOD_OFFSET) | + sq->pi << WQE_CSEG_WQE_INDEX_OFFSET); + /* There are 2 meters in one ASO cache line. */ + dseg_idx = aso_mtr->offset & 0x1; + wqe->aso_cseg.data_mask = + RTE_BE64(MLX5_IFC_FLOW_METER_PARAM_MASK << (32 * !dseg_idx)); + if (fm->is_enable) { + wqe->aso_dseg.mtrs[dseg_idx].cbs_cir = + fm->profile->srtcm_prm.cbs_cir; + wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = + fm->profile->srtcm_prm.ebs_eir; + } else { + wqe->aso_dseg.mtrs[dseg_idx].cbs_cir = + RTE_BE32(MLX5_IFC_FLOW_METER_DISABLE_CBS_CIR_VAL); + wqe->aso_dseg.mtrs[dseg_idx].ebs_eir = 0; + } + sq->head++; + sq->pi += 2;/* Each WQE contains 2 WQEBB's. */ + rte_io_wmb(); + sq->db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32(sq->pi); + rte_wmb(); + *sq->uar_addr = *(volatile uint64_t *)wqe; /* Assume 64 bit ARCH. */ + rte_wmb(); + return 1; +} + +static void +mlx5_aso_mtrs_status_update(struct mlx5_aso_sq *sq, uint16_t aso_mtrs_nums) +{ + uint16_t size = 1 << sq->log_desc_n; + uint16_t mask = size - 1; + uint16_t i; + struct mlx5_aso_mtr *aso_mtr = NULL; + uint8_t exp_state = ASO_METER_WAIT; + + for (i = 0; i < aso_mtrs_nums; ++i) { + aso_mtr = sq->elts[(sq->tail + i) & mask].mtr; + MLX5_ASSERT(aso_mtr); + __atomic_compare_exchange_n(&aso_mtr->state, + &exp_state, ASO_METER_READY, + false, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + } +} + +static void +mlx5_aso_mtr_completion_handle(struct mlx5_aso_sq *sq) +{ + struct mlx5_aso_cq *cq = &sq->cq; + volatile struct mlx5_cqe *restrict cqe; + const unsigned int cq_size = 1 << cq->log_desc_n; + const unsigned int mask = cq_size - 1; + uint32_t idx; + uint32_t next_idx = cq->cq_ci & mask; + const uint16_t max = (uint16_t)(sq->head - sq->tail); + uint16_t n = 0; + int ret; + + if (unlikely(!max)) + return; + do { + idx = next_idx; + next_idx = (cq->cq_ci + 1) & mask; + rte_prefetch0(&cq->cqes[next_idx]); + cqe = &cq->cqes[idx]; + ret = check_cqe(cqe, cq_size, cq->cq_ci); + /* + * Be sure owner read is done before any other cookie field or + * opaque field. + */ + rte_io_rmb(); + if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) { + if (likely(ret == MLX5_CQE_STATUS_HW_OWN)) + break; + mlx5_aso_cqe_err_handle(sq); + } else { + n++; + } + cq->cq_ci++; + } while (1); + if (likely(n)) { + mlx5_aso_mtrs_status_update(sq, n); + sq->tail += n; + rte_io_wmb(); + cq->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci); + } +} + +/** + * Update meter parameter by send WQE. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] priv + * Pointer to mlx5 private data structure. + * @param[in] fm + * Pointer to flow meter to be modified. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_aso_meter_update_by_wqe(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr) +{ + struct mlx5_aso_sq *sq = &sh->mtrmng->sq; + uint32_t poll_wqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; + + do { + mlx5_aso_mtr_completion_handle(sq); + if (mlx5_aso_mtr_sq_enqueue_single(sq, mtr)) + return 0; + /* Waiting for wqe resource. */ + usleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); + } while (--poll_wqe_times); + DRV_LOG(ERR, "Fail to send WQE for ASO meter %d", + mtr->fm.meter_id); + return -1; +} + +/** + * Wait for meter to be ready. + * + * @param[in] dev + * Pointer to Ethernet device. + * @param[in] priv + * Pointer to mlx5 private data structure. + * @param[in] fm + * Pointer to flow meter to be modified. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_aso_mtr_wait(struct mlx5_dev_ctx_shared *sh, + struct mlx5_aso_mtr *mtr) +{ + struct mlx5_aso_sq *sq = &sh->mtrmng->sq; + uint32_t poll_cqe_times = MLX5_MTR_POLL_WQE_CQE_TIMES; + + if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) == + ASO_METER_READY) + return 0; + do { + mlx5_aso_mtr_completion_handle(sq); + if (__atomic_load_n(&mtr->state, __ATOMIC_RELAXED) == + ASO_METER_READY) + return 0; + /* Waiting for CQE ready. */ + usleep(MLX5_ASO_WQE_CQE_RESPONSE_DELAY); + } while (--poll_cqe_times); + DRV_LOG(ERR, "Fail to poll CQE ready for ASO meter %d", + mtr->fm.meter_id); + return -1; +} diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index 9bac7d3a02..cc84cb3a52 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -10753,11 +10753,12 @@ flow_dv_translate(struct rte_eth_dev *dev, "meter not found " "or invalid parameters"); flow->meter = mtr_idx; - wks->fm = fm; + wks->mtr_idx = mtr_idx; } /* Set the meter action. */ if (!fm) { - fm = wks->fm; + fm = flow_dv_meter_find_by_idx(priv, + wks->mtr_idx); if (!fm) return rte_flow_error_set(error, rte_errno, diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index b17a71033d..31bbe00173 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -536,56 +536,79 @@ static int mlx5_flow_meter_action_modify(struct mlx5_priv *priv, struct mlx5_flow_meter_info *fm, const struct mlx5_flow_meter_srtcm_rfc2697_prm *srtcm, - uint64_t modify_bits, uint32_t active_state) + uint64_t modify_bits, uint32_t active_state, uint32_t is_enable) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER uint32_t in[MLX5_ST_SZ_DW(flow_meter_parameters)] = { 0 }; uint32_t *attr; struct mlx5dv_dr_flow_meter_attr mod_attr = { 0 }; int ret; + struct mlx5_aso_mtr *aso_mtr = NULL; uint32_t cbs_cir, ebs_eir, val; - /* Fill command parameters. */ - mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; - mod_attr.flow_meter_parameter = in; - mod_attr.flow_meter_parameter_sz = + if (priv->sh->meter_aso_en) { + fm->is_enable = !!is_enable; + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); + if (ret) + return ret; + ret = mlx5_aso_mtr_wait(priv->sh, aso_mtr); + if (ret) + return ret; + } else { + /* Fill command parameters. */ + mod_attr.reg_c_index = priv->mtr_color_reg - REG_C_0; + mod_attr.flow_meter_parameter = in; + mod_attr.flow_meter_parameter_sz = MLX5_ST_SZ_BYTES(flow_meter_parameters); - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) - mod_attr.active = !!active_state; - else - mod_attr.active = 0; - attr = in; - cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); - ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { - val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; - MLX5_SET(flow_meter_parameters, attr, cbs_exponent, val); - val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; - MLX5_SET(flow_meter_parameters, attr, cbs_mantissa, val); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { - val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & ASO_DSEG_EXP_MASK; - MLX5_SET(flow_meter_parameters, attr, cir_exponent, val); - val = cbs_cir & ASO_DSEG_MAN_MASK; - MLX5_SET(flow_meter_parameters, attr, cir_mantissa, val); - } - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { - val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & ASO_DSEG_EXP_MASK; - MLX5_SET(flow_meter_parameters, attr, ebs_exponent, val); - val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & ASO_DSEG_MAN_MASK; - MLX5_SET(flow_meter_parameters, attr, ebs_mantissa, val); - } - /* Apply modifications to meter only if it was created. */ - if (fm->mfts->meter_action) { - ret = mlx5_glue->dv_modify_flow_action_meter + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) + mod_attr.active = !!active_state; + else + mod_attr.active = 0; + attr = in; + cbs_cir = rte_be_to_cpu_32(srtcm->cbs_cir); + ebs_eir = rte_be_to_cpu_32(srtcm->ebs_eir); + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CBS) { + val = (cbs_cir >> ASO_DSEG_CBS_EXP_OFFSET) & + ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, + cbs_exponent, val); + val = (cbs_cir >> ASO_DSEG_CBS_MAN_OFFSET) & + ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, + cbs_mantissa, val); + } + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_CIR) { + val = (cbs_cir >> ASO_DSEG_CIR_EXP_OFFSET) & + ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, + cir_exponent, val); + val = cbs_cir & ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, + cir_mantissa, val); + } + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_EBS) { + val = (ebs_eir >> ASO_DSEG_EBS_EXP_OFFSET) & + ASO_DSEG_EXP_MASK; + MLX5_SET(flow_meter_parameters, attr, + ebs_exponent, val); + val = (ebs_eir >> ASO_DSEG_EBS_MAN_OFFSET) & + ASO_DSEG_MAN_MASK; + MLX5_SET(flow_meter_parameters, attr, + ebs_mantissa, val); + } + /* Apply modifications to meter only if it was created. */ + if (fm->mfts->meter_action) { + ret = mlx5_glue->dv_modify_flow_action_meter (fm->mfts->meter_action, &mod_attr, rte_cpu_to_be_64(modify_bits)); - if (ret) - return ret; + if (ret) + return ret; + } + /* Update succeedded modify meter parameters. */ + if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) + fm->active_state = !!active_state; } - /* Update succeedded modify meter parameters. */ - if (modify_bits & MLX5_FLOW_METER_OBJ_MODIFY_FIELD_ACTIVE) - fm->active_state = !!active_state; return 0; #else (void)priv; @@ -624,6 +647,7 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, struct mlx5_flow_meter_profile *fmp; struct mlx5_legacy_flow_meter *legacy_fm; struct mlx5_flow_meter_info *fm; + struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; const struct rte_flow_attr attr = { .ingress = 1, .egress = 1, @@ -632,6 +656,7 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, int ret; unsigned int i; struct mlx5_aso_mtr *aso_mtr; + union mlx5_l3t_data data; uint32_t mtr_idx; if (!priv->mtr_en) @@ -672,7 +697,6 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, fm->profile = fmp; memcpy(fm->action, params->action, sizeof(params->action)); fm->stats_mask = params->stats_mask; - /* Alloc policer counters. */ for (i = 0; i < RTE_DIM(fm->policer_stats.cnt); i++) { fm->policer_stats.cnt[i] = mlx5_counter_alloc(dev); @@ -689,10 +713,21 @@ mlx5_flow_meter_create(struct rte_eth_dev *dev, uint32_t meter_id, if (!priv->sh->meter_aso_en) TAILQ_INSERT_TAIL(fms, legacy_fm, next); fm->active_state = 1; /* Config meter starts as active. */ + fm->is_enable = 1; fm->shared = !!shared; fm->policer_stats.stats_mask = params->stats_mask; __atomic_add_fetch(&fm->profile->ref_cnt, 1, __ATOMIC_RELAXED); rte_spinlock_init(&fm->sl); + /* If ASO meter supported, allocate ASO flow meter. */ + if (priv->sh->meter_aso_en) { + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + ret = mlx5_aso_meter_update_by_wqe(priv->sh, aso_mtr); + if (ret) + goto error; + data.dword = mtr_idx; + if (mlx5_l3t_set_entry(mtrmng->mtr_idx_tbl, meter_id, &data)) + goto error; + } return 0; error: mlx5_flow_destroy_policer_rules(dev, fm, &attr); @@ -837,12 +872,12 @@ mlx5_flow_meter_modify_state(struct mlx5_priv *priv, int ret; if (new_state == MLX5_FLOW_METER_DISABLE) - ret = mlx5_flow_meter_action_modify(priv, fm, &srtcm, - modify_bits, 0); + ret = mlx5_flow_meter_action_modify(priv, fm, + &srtcm, modify_bits, 0, 0); else ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, - modify_bits, 0); + modify_bits, 0, 1); if (ret) return -rte_mtr_error_set(error, -ret, RTE_MTR_ERROR_TYPE_MTR_PARAMS, @@ -990,7 +1025,7 @@ mlx5_flow_meter_profile_update(struct rte_eth_dev *dev, if (fm->active_state == MLX5_FLOW_METER_DISABLE) return 0; ret = mlx5_flow_meter_action_modify(priv, fm, &fm->profile->srtcm_prm, - modify_bits, fm->active_state); + modify_bits, fm->active_state, 1); if (ret) { fm->profile = old_fmp; return -rte_mtr_error_set(error, -ret, @@ -1178,6 +1213,8 @@ mlx5_flow_meter_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg) * Pointer to mlx5_priv. * @param meter_id * Meter id. + * @param mtr_idx + * Pointer to Meter index. * * @return * Pointer to the profile found on success, NULL otherwise. @@ -1194,10 +1231,6 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, if (priv->sh->meter_aso_en) { rte_spinlock_lock(&mtrmng->mtrsl); - if (!mtrmng->n_valid) { - rte_spinlock_unlock(&mtrmng->mtrsl); - return NULL; - } if (mlx5_l3t_get_entry(mtrmng->mtr_idx_tbl, meter_id, &data) || !data.dword) { rte_spinlock_unlock(&mtrmng->mtrsl); @@ -1206,17 +1239,18 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, if (mtr_idx) *mtr_idx = data.dword; aso_mtr = mlx5_aso_meter_by_idx(priv, data.dword); + /* Remove reference taken by the mlx5_l3t_get_entry. */ mlx5_l3t_clear_entry(mtrmng->mtr_idx_tbl, meter_id); - if (meter_id == aso_mtr->fm.meter_id) { - rte_spinlock_unlock(&mtrmng->mtrsl); - return &aso_mtr->fm; - } + MLX5_ASSERT(meter_id == aso_mtr->fm.meter_id); rte_spinlock_unlock(&mtrmng->mtrsl); - } else { - TAILQ_FOREACH(legacy_fm, fms, next) - if (meter_id == legacy_fm->fm.meter_id) - return &legacy_fm->fm; + return &aso_mtr->fm; } + TAILQ_FOREACH(legacy_fm, fms, next) + if (meter_id == legacy_fm->fm.meter_id) { + if (mtr_idx) + *mtr_idx = legacy_fm->idx; + return &legacy_fm->fm; + } return NULL; } From patchwork Tue Dec 15 07:31:19 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Li Zhang X-Patchwork-Id: 85193 X-Patchwork-Delegate: rasland@nvidia.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 885F8A09E9; Tue, 15 Dec 2020 08:33:46 +0100 (CET) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 124FCCA02; Tue, 15 Dec 2020 08:31:54 +0100 (CET) Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id F13B3C9D0 for ; Tue, 15 Dec 2020 08:31:45 +0100 (CET) Received: from Internal Mail-Server by MTLPINE1 (envelope-from lizh@nvidia.com) with SMTP; 15 Dec 2020 09:31:41 +0200 Received: from nvidia.com (c-236-2-240-245.mtl.labs.mlnx [10.236.2.245]) by labmailer.mlnx (8.13.8/8.13.8) with ESMTP id 0BF7VOkK005436; Tue, 15 Dec 2020 09:31:41 +0200 From: Li Zhang To: dekelp@nvidia.com, orika@nvidia.com, viacheslavo@nvidia.com, matan@nvidia.com Cc: dev@dpdk.org, thomas@monjalon.net, rasland@nvidia.com, Shun Hao Date: Tue, 15 Dec 2020 09:31:19 +0200 Message-Id: <20201215073119.404947-9-lizh@nvidia.com> X-Mailer: git-send-email 2.27.0 In-Reply-To: <20201215073119.404947-1-lizh@nvidia.com> References: <20201215073119.404947-1-lizh@nvidia.com> MIME-Version: 1.0 Subject: [dpdk-dev] [RFC 8/8] net/mlx5: add support of ASO meter action X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" When ASO action is available, use it as the meter action Signed-off-by: Shun Hao Signed-off-by: Li Zhang --- drivers/net/mlx5/linux/mlx5_os.c | 4 +- drivers/net/mlx5/mlx5.c | 8 ++ drivers/net/mlx5/mlx5.h | 18 ++--- drivers/net/mlx5/mlx5_flow.c | 66 +++++++++++----- drivers/net/mlx5/mlx5_flow_dv.c | 74 ++++++++++-------- drivers/net/mlx5/mlx5_flow_meter.c | 118 ++++++++++++++++++----------- 6 files changed, 184 insertions(+), 104 deletions(-) diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 01b019c28d..28e383c7e2 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -1233,7 +1233,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, "required for coalescing is %d bytes", config->hca_attr.lro_min_mss_size); } -#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) +#if defined(HAVE_MLX5DV_DR) && \ + (defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) || \ + defined(HAVE_MLX5_DR_CREATE_ACTION_ASO)) if (config->hca_attr.qos.sup && config->hca_attr.qos.srtcm_sup && config->dv_flow_en) { diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 9d77265e2e..224c8e3965 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -584,13 +584,21 @@ static void mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh) { struct mlx5_aso_mtr_pool *mtr_pool; + struct mlx5_aso_mtr *aso_mtr; struct mlx5_aso_mtr_pools_mng *mtrmng = sh->mtrmng; uint32_t idx; + int i; mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER); idx = mtrmng->n_valid; while (idx--) { mtr_pool = mtrmng->pools[idx]; + for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) { + aso_mtr = &mtr_pool->mtrs[i]; + if (aso_mtr->fm.meter_action) + claim_zero(mlx5_glue->destroy_flow_action + (aso_mtr->fm.meter_action)); + } claim_zero(mlx5_devx_cmd_destroy (mtr_pool->devx_obj)); mtrmng->n_valid--; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 73d3698f1d..b4bd76e792 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -645,8 +645,6 @@ struct mlx5_meter_domains_infos { /**< Drop action as not matched. */ void *count_actns[RTE_MTR_DROPPED + 1]; /**< Counters for match and unmatched statistics. */ - void *meter_action; - /**< Flow meter action. */ }; /* Meter parameter structure. */ @@ -697,6 +695,8 @@ struct mlx5_flow_meter_info { /**< Meter shared or not. */ uint32_t is_enable:1; /**< Meter disable/enable state. */ + void *meter_action; + /**< Flow meter action. */ }; /* RFC2697 parameter structure. */ @@ -800,7 +800,7 @@ struct mlx5_flow_tbl_resource { /* Tables for metering splits should be added here. */ #define MLX5_FLOW_TABLE_LEVEL_SUFFIX (MLX5_MAX_TABLES - 3) #define MLX5_FLOW_TABLE_LEVEL_METER (MLX5_MAX_TABLES - 4) -#define MLX5_MAX_TABLES_EXTERNAL MLX5_FLOW_TABLE_LEVEL_METER +#define MLX5_MAX_TABLES_EXTERNAL (MLX5_FLOW_TABLE_LEVEL_METER - 1) #define MLX5_MAX_TABLES_FDB UINT16_MAX #define MLX5_FLOW_TABLE_FACTOR 10 @@ -1458,13 +1458,11 @@ int mlx5_pmd_socket_init(void); int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg); struct mlx5_flow_meter_info *mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, uint32_t *mtr_idx); -struct mlx5_flow_meter_info *mlx5_flow_meter_attach - (struct mlx5_priv *priv, - uint32_t meter_id, - const struct rte_flow_attr *attr, - uint32_t *mtr_idx, - struct rte_flow_error *error); -void mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm); +uint32_t mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, + const struct rte_flow_attr *attr, + struct rte_flow_error *error); +void mlx5_flow_meter_detach(struct mlx5_priv *priv, + struct mlx5_flow_meter_info *fm); /* mlx5_os.c */ struct rte_pci_driver; diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 2f047bbe01..0142b546df 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -4282,6 +4282,7 @@ flow_create_split_inner(struct rte_eth_dev *dev, */ static int flow_meter_split_prep(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, const struct rte_flow_item items[], struct rte_flow_item sfx_items[], const struct rte_flow_action actions[], @@ -4300,33 +4301,44 @@ flow_meter_split_prep(struct rte_eth_dev *dev, uint32_t tag_id = 0; uint32_t reg_id = 0; bool copy_vlan = false; + struct rte_flow_action *hw_mtr_action; + struct rte_flow_action_jump *jump_data; + /* For ASO meter, meter must be before tag in TX direction. */ + bool mtr_first = priv->sh->meter_aso_en && + (attr->egress || + (attr->transfer && priv->representor_id != -1)); + struct rte_flow_action *action_pre_head = + mtr_first ? actions_pre++ : NULL; /* Prepare the actions for prefix and suffix flow. */ for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) { - struct rte_flow_action **action_cur = NULL; + struct rte_flow_action *action_cur = NULL; switch (actions->type) { case RTE_FLOW_ACTION_TYPE_METER: - /* Add the extra tag action first. */ - tag_action = actions_pre; + if (mtr_first) { + action_cur = action_pre_head; + tag_action = actions_pre++; + } else { + tag_action = actions_pre++; + action_cur = actions_pre++; + } tag_action->type = (enum rte_flow_action_type) MLX5_RTE_FLOW_ACTION_TYPE_TAG; - actions_pre++; - action_cur = &actions_pre; break; case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP: case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP: - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_ENCAP: raw_encap = actions->conf; if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_RAW_DECAP: raw_decap = actions->conf; if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE) - action_cur = &actions_pre; + action_cur = actions_pre++; break; case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN: case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID: @@ -4336,14 +4348,30 @@ flow_meter_split_prep(struct rte_eth_dev *dev, break; } if (!action_cur) - action_cur = &actions_sfx; - memcpy(*action_cur, actions, sizeof(struct rte_flow_action)); - (*action_cur)++; + action_cur = actions_sfx++; + memcpy(action_cur, actions, sizeof(struct rte_flow_action)); } /* Add end action to the actions. */ actions_sfx->type = RTE_FLOW_ACTION_TYPE_END; - actions_pre->type = RTE_FLOW_ACTION_TYPE_END; - actions_pre++; + if (priv->sh->meter_aso_en) { + /** For ASO meter, need to add an extra jump action explicitly, + * to jump from meter to policer table. + */ + hw_mtr_action = actions_pre; + hw_mtr_action->type = RTE_FLOW_ACTION_TYPE_JUMP; + actions_pre++; + actions_pre->type = RTE_FLOW_ACTION_TYPE_END; + actions_pre++; + jump_data = (void *)actions_pre; + jump_data->group = attr->transfer ? + (MLX5_FLOW_TABLE_LEVEL_METER - 1) : + MLX5_FLOW_TABLE_LEVEL_METER; + hw_mtr_action->conf = jump_data; + actions_pre++; + } else { + actions_pre->type = RTE_FLOW_ACTION_TYPE_END; + actions_pre++; + } /* Set the tag. */ set_tag = (void *)actions_pre; reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error); @@ -5116,12 +5144,16 @@ flow_create_split_meter(struct rte_eth_dev *dev, size_t item_size; int actions_n = 0; int ret; + uint8_t pre_action_num; if (priv->mtr_en) actions_n = flow_check_meter_action(actions, &mtr); if (mtr) { /* The five prefix actions: meter, decap, encap, tag, end. */ - act_size = sizeof(struct rte_flow_action) * (actions_n + 5) + + /* For aso meter, need 2 more space for jump. */ + pre_action_num = priv->sh->meter_aso_en ? 7 : 5; + act_size = sizeof(struct rte_flow_action) * + (actions_n + pre_action_num) + sizeof(struct mlx5_rte_flow_action_set_tag); /* tag, vlan, port id, end. */ #define METER_SUFFIX_ITEM 4 @@ -5137,9 +5169,9 @@ flow_create_split_meter(struct rte_eth_dev *dev, sfx_items = (struct rte_flow_item *)((char *)sfx_actions + act_size); pre_actions = sfx_actions + actions_n; - mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items, - actions, sfx_actions, - pre_actions); + mtr_tag_id = flow_meter_split_prep(dev, &sfx_attr, items, + sfx_items, actions, + sfx_actions, pre_actions); if (!mtr_tag_id) { ret = -rte_errno; goto exit; diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c index cc84cb3a52..b0803e4847 100644 --- a/drivers/net/mlx5/mlx5_flow_dv.c +++ b/drivers/net/mlx5/mlx5_flow_dv.c @@ -4213,9 +4213,11 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev, return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Meter not found"); - if (fm->ref_cnt && (!(fm->transfer == attr->transfer || + /* aso meter can always be shared by different domains */ + if (fm->ref_cnt && !priv->sh->meter_aso_en && + !(fm->transfer == attr->transfer || (!fm->ingress && !attr->ingress && attr->egress) || - (!fm->egress && !attr->egress && attr->ingress)))) + (!fm->egress && !attr->egress && attr->ingress))) return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Flow attributes are either invalid " @@ -5394,6 +5396,8 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev) struct mlx5_aso_mtr *mtr_free = NULL; struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng; struct mlx5_aso_mtr_pool *pool; + struct rte_flow_error error; + uint8_t reg_id; uint32_t mtr_idx = 0; if (!priv->config.devx) { @@ -5416,6 +5420,20 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev) struct mlx5_aso_mtr_pool, mtrs[mtr_free->offset]); mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset); + if (!mtr_free->fm.meter_action) { + reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error); + mtr_free->fm.meter_action = + mlx5_glue->dv_create_flow_action_aso + (priv->sh->rx_domain, + pool->devx_obj->obj, + mtr_free->offset, + (1 << MLX5_FLOW_COLOR_GREEN), + reg_id - REG_C_0); + if (!mtr_free->fm.meter_action) { + flow_dv_aso_mtr_release_to_pool(dev, mtr_idx); + return 0; + } + } return mtr_idx; } @@ -10317,7 +10335,6 @@ flow_dv_translate(struct rte_eth_dev *dev, const struct rte_flow_action *found_action = NULL; struct mlx5_flow_meter_info *fm = NULL; uint32_t jump_group = 0; - uint32_t mtr_idx; if (!mlx5_flow_os_action_supported(action_type)) return rte_flow_error_set(error, ENOTSUP, @@ -10743,32 +10760,23 @@ flow_dv_translate(struct rte_eth_dev *dev, case RTE_FLOW_ACTION_TYPE_METER: mtr = actions->conf; if (!flow->meter) { - fm = mlx5_flow_meter_attach(priv, mtr->mtr_id, - attr, &mtr_idx, error); - if (!fm) - return rte_flow_error_set(error, - rte_errno, - RTE_FLOW_ERROR_TYPE_ACTION, - NULL, - "meter not found " - "or invalid parameters"); - flow->meter = mtr_idx; - wks->mtr_idx = mtr_idx; + flow->meter = + mlx5_flow_meter_attach(priv, + mtr->mtr_id, + attr, + error); + if (!flow->meter) + return -rte_errno; + wks->mtr_idx = flow->meter; } - /* Set the meter action. */ - if (!fm) { - fm = flow_dv_meter_find_by_idx(priv, - wks->mtr_idx); - if (!fm) - return rte_flow_error_set(error, - rte_errno, + fm = flow_dv_meter_find_by_idx(priv, wks->mtr_idx); + if (!fm) + return rte_flow_error_set(error, ENOENT, RTE_FLOW_ERROR_TYPE_ACTION, NULL, - "meter not found " - "or invalid parameters"); - } - dev_flow->dv.actions[actions_n++] = - fm->mfts->meter_action; + "failed to get flow meter."); + /* Set the meter action. */ + dev_flow->dv.actions[actions_n++] = fm->meter_action; action_flags |= MLX5_FLOW_ACTION_METER; break; case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP: @@ -11888,7 +11896,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow) fm = flow_dv_meter_find_by_idx(priv, flow->meter); if (fm) - mlx5_flow_meter_detach(fm); + mlx5_flow_meter_detach(priv, fm); flow->meter = 0; } if (flow->age) @@ -12678,10 +12686,12 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_dev_ctx_shared *sh = priv->sh; struct mlx5_flow_dv_match_params mask = { - .size = sizeof(mask.buf), + .size = sizeof(mask.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5dv_flow_matcher_attr dv_attr = { .type = IBV_FLOW_ATTR_NORMAL, @@ -12901,10 +12911,12 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter_info *fm, uint8_t mtr_reg_c) { struct mlx5_flow_dv_match_params matcher = { - .size = sizeof(matcher.buf), + .size = sizeof(matcher.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_flow_dv_match_params value = { - .size = sizeof(value.buf), + .size = sizeof(value.buf) - + MLX5_ST_SZ_BYTES(fte_match_set_misc4), }; struct mlx5_meter_domains_infos *mtb = fm->mfts; void *actions[METER_ACTIONS]; diff --git a/drivers/net/mlx5/mlx5_flow_meter.c b/drivers/net/mlx5/mlx5_flow_meter.c index 31bbe00173..b01aae2301 100644 --- a/drivers/net/mlx5/mlx5_flow_meter.c +++ b/drivers/net/mlx5/mlx5_flow_meter.c @@ -598,9 +598,9 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv, ebs_mantissa, val); } /* Apply modifications to meter only if it was created. */ - if (fm->mfts->meter_action) { + if (fm->meter_action) { ret = mlx5_glue->dv_modify_flow_action_meter - (fm->mfts->meter_action, &mod_attr, + (fm->meter_action, &mod_attr, rte_cpu_to_be_64(modify_bits)); if (ret) return ret; @@ -1268,74 +1268,102 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id, * @param [out] error * Pointer to error structure. * - * @return the flow meter pointer, NULL otherwise. + * @return + * The meter idx on success, 0 otherwise and rte_errno is set. */ -struct mlx5_flow_meter_info * -mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, - const struct rte_flow_attr *attr, uint32_t *mtr_idx, - struct rte_flow_error *error) +uint32_t mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id, + const struct rte_flow_attr *attr, + struct rte_flow_error *error) { struct mlx5_flow_meter_info *fm; + struct mlx5_aso_mtr *aso_mtr; + uint32_t mtr_idx; int ret = 0; - fm = mlx5_flow_meter_find(priv, meter_id, mtr_idx); - if (fm == NULL) { + fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx); + if (!fm) { rte_flow_error_set(error, ENOENT, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "Meter object id not valid"); - return fm; + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "Meter not found "); + return 0; } - rte_spinlock_lock(&fm->sl); - if (fm->mfts->meter_action) { - if (fm->shared && - attr->transfer == fm->transfer && - attr->ingress == fm->ingress && - attr->egress == fm->egress) + if (priv->sh->meter_aso_en) { + aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm); + if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) { + rte_flow_error_set(error, ENOENT, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "Timeout in meter configuration"); + return 0; + } + rte_spinlock_lock(&fm->sl); + if (fm->shared || !fm->ref_cnt) { fm->ref_cnt++; - else + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter cannot be shared"); ret = -1; + } + rte_spinlock_unlock(&fm->sl); } else { - fm->ingress = attr->ingress; - fm->egress = attr->egress; - fm->transfer = attr->transfer; - fm->ref_cnt = 1; - /* This also creates the meter object. */ - fm->mfts->meter_action = mlx5_flow_meter_action_create(priv, - fm); - if (!fm->mfts->meter_action) { - fm->ref_cnt = 0; - fm->ingress = 0; - fm->egress = 0; - fm->transfer = 0; - ret = -1; - DRV_LOG(ERR, "Meter action create failed."); + rte_spinlock_lock(&fm->sl); + if (fm->meter_action) { + if (fm->shared && + attr->transfer == fm->transfer && + attr->ingress == fm->ingress && + attr->egress == fm->egress) { + fm->ref_cnt++; + } else { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + fm->shared ? + "Meter attr not match." : + "Meter cannot be shared."); + ret = -1; + } + } else { + fm->ingress = attr->ingress; + fm->egress = attr->egress; + fm->transfer = attr->transfer; + fm->ref_cnt = 1; + /* This also creates the meter object. */ + fm->meter_action = mlx5_flow_meter_action_create(priv, + fm); + if (!fm->meter_action) { + fm->ref_cnt = 0; + fm->ingress = 0; + fm->egress = 0; + fm->transfer = 0; + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, + "Meter action create failed."); + ret = -1; + } } + rte_spinlock_unlock(&fm->sl); } - rte_spinlock_unlock(&fm->sl); - if (ret) - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - fm->mfts->meter_action ? - "Meter attr not match" : - "Meter action create failed"); - return ret ? NULL : fm; + return (ret < 0) ? 0 : mtr_idx; } /** * Detach meter from flow. * + * @param [in] priv + * Pointer to mlx5 private data. * @param [in] fm * Pointer to flow meter. */ void -mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm) +mlx5_flow_meter_detach(struct mlx5_priv *priv, + struct mlx5_flow_meter_info *fm) { #ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER rte_spinlock_lock(&fm->sl); MLX5_ASSERT(fm->ref_cnt); - if (--fm->ref_cnt == 0) { - mlx5_glue->destroy_flow_action(fm->mfts->meter_action); - fm->mfts->meter_action = NULL; + if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) { + mlx5_glue->destroy_flow_action(fm->meter_action); + fm->meter_action = NULL; fm->ingress = 0; fm->egress = 0; fm->transfer = 0;