@@ -1233,7 +1233,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
"required for coalescing is %d bytes",
config->hca_attr.lro_min_mss_size);
}
-#if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER)
+#if defined(HAVE_MLX5DV_DR) && \
+ (defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) || \
+ defined(HAVE_MLX5_DR_CREATE_ACTION_ASO))
if (config->hca_attr.qos.sup &&
config->hca_attr.qos.srtcm_sup &&
config->dv_flow_en) {
@@ -584,13 +584,21 @@ static void
mlx5_aso_flow_mtrs_mng_close(struct mlx5_dev_ctx_shared *sh)
{
struct mlx5_aso_mtr_pool *mtr_pool;
+ struct mlx5_aso_mtr *aso_mtr;
struct mlx5_aso_mtr_pools_mng *mtrmng = sh->mtrmng;
uint32_t idx;
+ int i;
mlx5_aso_queue_uninit(sh, ASO_OPC_MOD_POLICER);
idx = mtrmng->n_valid;
while (idx--) {
mtr_pool = mtrmng->pools[idx];
+ for (i = 0; i < MLX5_ASO_MTRS_PER_POOL; i++) {
+ aso_mtr = &mtr_pool->mtrs[i];
+ if (aso_mtr->fm.meter_action)
+ claim_zero(mlx5_glue->destroy_flow_action
+ (aso_mtr->fm.meter_action));
+ }
claim_zero(mlx5_devx_cmd_destroy
(mtr_pool->devx_obj));
mtrmng->n_valid--;
@@ -645,8 +645,6 @@ struct mlx5_meter_domains_infos {
/**< Drop action as not matched. */
void *count_actns[RTE_MTR_DROPPED + 1];
/**< Counters for match and unmatched statistics. */
- void *meter_action;
- /**< Flow meter action. */
};
/* Meter parameter structure. */
@@ -697,6 +695,8 @@ struct mlx5_flow_meter_info {
/**< Meter shared or not. */
uint32_t is_enable:1;
/**< Meter disable/enable state. */
+ void *meter_action;
+ /**< Flow meter action. */
};
/* RFC2697 parameter structure. */
@@ -800,7 +800,7 @@ struct mlx5_flow_tbl_resource {
/* Tables for metering splits should be added here. */
#define MLX5_FLOW_TABLE_LEVEL_SUFFIX (MLX5_MAX_TABLES - 3)
#define MLX5_FLOW_TABLE_LEVEL_METER (MLX5_MAX_TABLES - 4)
-#define MLX5_MAX_TABLES_EXTERNAL MLX5_FLOW_TABLE_LEVEL_METER
+#define MLX5_MAX_TABLES_EXTERNAL (MLX5_FLOW_TABLE_LEVEL_METER - 1)
#define MLX5_MAX_TABLES_FDB UINT16_MAX
#define MLX5_FLOW_TABLE_FACTOR 10
@@ -1458,13 +1458,11 @@ int mlx5_pmd_socket_init(void);
int mlx5_flow_meter_ops_get(struct rte_eth_dev *dev, void *arg);
struct mlx5_flow_meter_info *mlx5_flow_meter_find(struct mlx5_priv *priv,
uint32_t meter_id, uint32_t *mtr_idx);
-struct mlx5_flow_meter_info *mlx5_flow_meter_attach
- (struct mlx5_priv *priv,
- uint32_t meter_id,
- const struct rte_flow_attr *attr,
- uint32_t *mtr_idx,
- struct rte_flow_error *error);
-void mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm);
+uint32_t mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error);
+void mlx5_flow_meter_detach(struct mlx5_priv *priv,
+ struct mlx5_flow_meter_info *fm);
/* mlx5_os.c */
struct rte_pci_driver;
@@ -4282,6 +4282,7 @@ flow_create_split_inner(struct rte_eth_dev *dev,
*/
static int
flow_meter_split_prep(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
struct rte_flow_item sfx_items[],
const struct rte_flow_action actions[],
@@ -4300,33 +4301,44 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
uint32_t tag_id = 0;
uint32_t reg_id = 0;
bool copy_vlan = false;
+ struct rte_flow_action *hw_mtr_action;
+ struct rte_flow_action_jump *jump_data;
+ /* For ASO meter, meter must be before tag in TX direction. */
+ bool mtr_first = priv->sh->meter_aso_en &&
+ (attr->egress ||
+ (attr->transfer && priv->representor_id != -1));
+ struct rte_flow_action *action_pre_head =
+ mtr_first ? actions_pre++ : NULL;
/* Prepare the actions for prefix and suffix flow. */
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
- struct rte_flow_action **action_cur = NULL;
+ struct rte_flow_action *action_cur = NULL;
switch (actions->type) {
case RTE_FLOW_ACTION_TYPE_METER:
- /* Add the extra tag action first. */
- tag_action = actions_pre;
+ if (mtr_first) {
+ action_cur = action_pre_head;
+ tag_action = actions_pre++;
+ } else {
+ tag_action = actions_pre++;
+ action_cur = actions_pre++;
+ }
tag_action->type = (enum rte_flow_action_type)
MLX5_RTE_FLOW_ACTION_TYPE_TAG;
- actions_pre++;
- action_cur = &actions_pre;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
- action_cur = &actions_pre;
+ action_cur = actions_pre++;
break;
case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
raw_encap = actions->conf;
if (raw_encap->size < MLX5_ENCAPSULATION_DECISION_SIZE)
- action_cur = &actions_pre;
+ action_cur = actions_pre++;
break;
case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
raw_decap = actions->conf;
if (raw_decap->size > MLX5_ENCAPSULATION_DECISION_SIZE)
- action_cur = &actions_pre;
+ action_cur = actions_pre++;
break;
case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
@@ -4336,14 +4348,30 @@ flow_meter_split_prep(struct rte_eth_dev *dev,
break;
}
if (!action_cur)
- action_cur = &actions_sfx;
- memcpy(*action_cur, actions, sizeof(struct rte_flow_action));
- (*action_cur)++;
+ action_cur = actions_sfx++;
+ memcpy(action_cur, actions, sizeof(struct rte_flow_action));
}
/* Add end action to the actions. */
actions_sfx->type = RTE_FLOW_ACTION_TYPE_END;
- actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
- actions_pre++;
+ if (priv->sh->meter_aso_en) {
+ /** For ASO meter, need to add an extra jump action explicitly,
+ * to jump from meter to policer table.
+ */
+ hw_mtr_action = actions_pre;
+ hw_mtr_action->type = RTE_FLOW_ACTION_TYPE_JUMP;
+ actions_pre++;
+ actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
+ actions_pre++;
+ jump_data = (void *)actions_pre;
+ jump_data->group = attr->transfer ?
+ (MLX5_FLOW_TABLE_LEVEL_METER - 1) :
+ MLX5_FLOW_TABLE_LEVEL_METER;
+ hw_mtr_action->conf = jump_data;
+ actions_pre++;
+ } else {
+ actions_pre->type = RTE_FLOW_ACTION_TYPE_END;
+ actions_pre++;
+ }
/* Set the tag. */
set_tag = (void *)actions_pre;
reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_SFX, 0, &error);
@@ -5116,12 +5144,16 @@ flow_create_split_meter(struct rte_eth_dev *dev,
size_t item_size;
int actions_n = 0;
int ret;
+ uint8_t pre_action_num;
if (priv->mtr_en)
actions_n = flow_check_meter_action(actions, &mtr);
if (mtr) {
/* The five prefix actions: meter, decap, encap, tag, end. */
- act_size = sizeof(struct rte_flow_action) * (actions_n + 5) +
+ /* For aso meter, need 2 more space for jump. */
+ pre_action_num = priv->sh->meter_aso_en ? 7 : 5;
+ act_size = sizeof(struct rte_flow_action) *
+ (actions_n + pre_action_num) +
sizeof(struct mlx5_rte_flow_action_set_tag);
/* tag, vlan, port id, end. */
#define METER_SUFFIX_ITEM 4
@@ -5137,9 +5169,9 @@ flow_create_split_meter(struct rte_eth_dev *dev,
sfx_items = (struct rte_flow_item *)((char *)sfx_actions +
act_size);
pre_actions = sfx_actions + actions_n;
- mtr_tag_id = flow_meter_split_prep(dev, items, sfx_items,
- actions, sfx_actions,
- pre_actions);
+ mtr_tag_id = flow_meter_split_prep(dev, &sfx_attr, items,
+ sfx_items, actions,
+ sfx_actions, pre_actions);
if (!mtr_tag_id) {
ret = -rte_errno;
goto exit;
@@ -4213,9 +4213,11 @@ mlx5_flow_validate_action_meter(struct rte_eth_dev *dev,
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"Meter not found");
- if (fm->ref_cnt && (!(fm->transfer == attr->transfer ||
+ /* aso meter can always be shared by different domains */
+ if (fm->ref_cnt && !priv->sh->meter_aso_en &&
+ !(fm->transfer == attr->transfer ||
(!fm->ingress && !attr->ingress && attr->egress) ||
- (!fm->egress && !attr->egress && attr->ingress))))
+ (!fm->egress && !attr->egress && attr->ingress)))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"Flow attributes are either invalid "
@@ -5394,6 +5396,8 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev)
struct mlx5_aso_mtr *mtr_free = NULL;
struct mlx5_aso_mtr_pools_mng *mtrmng = priv->sh->mtrmng;
struct mlx5_aso_mtr_pool *pool;
+ struct rte_flow_error error;
+ uint8_t reg_id;
uint32_t mtr_idx = 0;
if (!priv->config.devx) {
@@ -5416,6 +5420,20 @@ flow_dv_mtr_alloc(struct rte_eth_dev *dev)
struct mlx5_aso_mtr_pool,
mtrs[mtr_free->offset]);
mtr_idx = MLX5_MAKE_MTR_IDX(pool->index, mtr_free->offset);
+ if (!mtr_free->fm.meter_action) {
+ reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, &error);
+ mtr_free->fm.meter_action =
+ mlx5_glue->dv_create_flow_action_aso
+ (priv->sh->rx_domain,
+ pool->devx_obj->obj,
+ mtr_free->offset,
+ (1 << MLX5_FLOW_COLOR_GREEN),
+ reg_id - REG_C_0);
+ if (!mtr_free->fm.meter_action) {
+ flow_dv_aso_mtr_release_to_pool(dev, mtr_idx);
+ return 0;
+ }
+ }
return mtr_idx;
}
@@ -10317,7 +10335,6 @@ flow_dv_translate(struct rte_eth_dev *dev,
const struct rte_flow_action *found_action = NULL;
struct mlx5_flow_meter_info *fm = NULL;
uint32_t jump_group = 0;
- uint32_t mtr_idx;
if (!mlx5_flow_os_action_supported(action_type))
return rte_flow_error_set(error, ENOTSUP,
@@ -10743,32 +10760,23 @@ flow_dv_translate(struct rte_eth_dev *dev,
case RTE_FLOW_ACTION_TYPE_METER:
mtr = actions->conf;
if (!flow->meter) {
- fm = mlx5_flow_meter_attach(priv, mtr->mtr_id,
- attr, &mtr_idx, error);
- if (!fm)
- return rte_flow_error_set(error,
- rte_errno,
- RTE_FLOW_ERROR_TYPE_ACTION,
- NULL,
- "meter not found "
- "or invalid parameters");
- flow->meter = mtr_idx;
- wks->mtr_idx = mtr_idx;
+ flow->meter =
+ mlx5_flow_meter_attach(priv,
+ mtr->mtr_id,
+ attr,
+ error);
+ if (!flow->meter)
+ return -rte_errno;
+ wks->mtr_idx = flow->meter;
}
- /* Set the meter action. */
- if (!fm) {
- fm = flow_dv_meter_find_by_idx(priv,
- wks->mtr_idx);
- if (!fm)
- return rte_flow_error_set(error,
- rte_errno,
+ fm = flow_dv_meter_find_by_idx(priv, wks->mtr_idx);
+ if (!fm)
+ return rte_flow_error_set(error, ENOENT,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
- "meter not found "
- "or invalid parameters");
- }
- dev_flow->dv.actions[actions_n++] =
- fm->mfts->meter_action;
+ "failed to get flow meter.");
+ /* Set the meter action. */
+ dev_flow->dv.actions[actions_n++] = fm->meter_action;
action_flags |= MLX5_FLOW_ACTION_METER;
break;
case RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP:
@@ -11888,7 +11896,7 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
fm = flow_dv_meter_find_by_idx(priv, flow->meter);
if (fm)
- mlx5_flow_meter_detach(fm);
+ mlx5_flow_meter_detach(priv, fm);
flow->meter = 0;
}
if (flow->age)
@@ -12678,10 +12686,12 @@ flow_dv_prepare_mtr_tables(struct rte_eth_dev *dev,
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_dev_ctx_shared *sh = priv->sh;
struct mlx5_flow_dv_match_params mask = {
- .size = sizeof(mask.buf),
+ .size = sizeof(mask.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
};
struct mlx5_flow_dv_match_params value = {
- .size = sizeof(value.buf),
+ .size = sizeof(value.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
};
struct mlx5dv_flow_matcher_attr dv_attr = {
.type = IBV_FLOW_ATTR_NORMAL,
@@ -12901,10 +12911,12 @@ flow_dv_create_policer_forward_rule(struct mlx5_flow_meter_info *fm,
uint8_t mtr_reg_c)
{
struct mlx5_flow_dv_match_params matcher = {
- .size = sizeof(matcher.buf),
+ .size = sizeof(matcher.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
};
struct mlx5_flow_dv_match_params value = {
- .size = sizeof(value.buf),
+ .size = sizeof(value.buf) -
+ MLX5_ST_SZ_BYTES(fte_match_set_misc4),
};
struct mlx5_meter_domains_infos *mtb = fm->mfts;
void *actions[METER_ACTIONS];
@@ -598,9 +598,9 @@ mlx5_flow_meter_action_modify(struct mlx5_priv *priv,
ebs_mantissa, val);
}
/* Apply modifications to meter only if it was created. */
- if (fm->mfts->meter_action) {
+ if (fm->meter_action) {
ret = mlx5_glue->dv_modify_flow_action_meter
- (fm->mfts->meter_action, &mod_attr,
+ (fm->meter_action, &mod_attr,
rte_cpu_to_be_64(modify_bits));
if (ret)
return ret;
@@ -1268,74 +1268,102 @@ mlx5_flow_meter_find(struct mlx5_priv *priv, uint32_t meter_id,
* @param [out] error
* Pointer to error structure.
*
- * @return the flow meter pointer, NULL otherwise.
+ * @return
+ * The meter idx on success, 0 otherwise and rte_errno is set.
*/
-struct mlx5_flow_meter_info *
-mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id,
- const struct rte_flow_attr *attr, uint32_t *mtr_idx,
- struct rte_flow_error *error)
+uint32_t mlx5_flow_meter_attach(struct mlx5_priv *priv, uint32_t meter_id,
+ const struct rte_flow_attr *attr,
+ struct rte_flow_error *error)
{
struct mlx5_flow_meter_info *fm;
+ struct mlx5_aso_mtr *aso_mtr;
+ uint32_t mtr_idx;
int ret = 0;
- fm = mlx5_flow_meter_find(priv, meter_id, mtr_idx);
- if (fm == NULL) {
+ fm = mlx5_flow_meter_find(priv, meter_id, &mtr_idx);
+ if (!fm) {
rte_flow_error_set(error, ENOENT,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- "Meter object id not valid");
- return fm;
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL, "Meter not found ");
+ return 0;
}
- rte_spinlock_lock(&fm->sl);
- if (fm->mfts->meter_action) {
- if (fm->shared &&
- attr->transfer == fm->transfer &&
- attr->ingress == fm->ingress &&
- attr->egress == fm->egress)
+ if (priv->sh->meter_aso_en) {
+ aso_mtr = container_of(fm, struct mlx5_aso_mtr, fm);
+ if (mlx5_aso_mtr_wait(priv->sh, aso_mtr)) {
+ rte_flow_error_set(error, ENOENT,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+ NULL,
+ "Timeout in meter configuration");
+ return 0;
+ }
+ rte_spinlock_lock(&fm->sl);
+ if (fm->shared || !fm->ref_cnt) {
fm->ref_cnt++;
- else
+ } else {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "Meter cannot be shared");
ret = -1;
+ }
+ rte_spinlock_unlock(&fm->sl);
} else {
- fm->ingress = attr->ingress;
- fm->egress = attr->egress;
- fm->transfer = attr->transfer;
- fm->ref_cnt = 1;
- /* This also creates the meter object. */
- fm->mfts->meter_action = mlx5_flow_meter_action_create(priv,
- fm);
- if (!fm->mfts->meter_action) {
- fm->ref_cnt = 0;
- fm->ingress = 0;
- fm->egress = 0;
- fm->transfer = 0;
- ret = -1;
- DRV_LOG(ERR, "Meter action create failed.");
+ rte_spinlock_lock(&fm->sl);
+ if (fm->meter_action) {
+ if (fm->shared &&
+ attr->transfer == fm->transfer &&
+ attr->ingress == fm->ingress &&
+ attr->egress == fm->egress) {
+ fm->ref_cnt++;
+ } else {
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ fm->shared ?
+ "Meter attr not match." :
+ "Meter cannot be shared.");
+ ret = -1;
+ }
+ } else {
+ fm->ingress = attr->ingress;
+ fm->egress = attr->egress;
+ fm->transfer = attr->transfer;
+ fm->ref_cnt = 1;
+ /* This also creates the meter object. */
+ fm->meter_action = mlx5_flow_meter_action_create(priv,
+ fm);
+ if (!fm->meter_action) {
+ fm->ref_cnt = 0;
+ fm->ingress = 0;
+ fm->egress = 0;
+ fm->transfer = 0;
+ rte_flow_error_set(error, EINVAL,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "Meter action create failed.");
+ ret = -1;
+ }
}
+ rte_spinlock_unlock(&fm->sl);
}
- rte_spinlock_unlock(&fm->sl);
- if (ret)
- rte_flow_error_set(error, EINVAL,
- RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- fm->mfts->meter_action ?
- "Meter attr not match" :
- "Meter action create failed");
- return ret ? NULL : fm;
+ return (ret < 0) ? 0 : mtr_idx;
}
/**
* Detach meter from flow.
*
+ * @param [in] priv
+ * Pointer to mlx5 private data.
* @param [in] fm
* Pointer to flow meter.
*/
void
-mlx5_flow_meter_detach(struct mlx5_flow_meter_info *fm)
+mlx5_flow_meter_detach(struct mlx5_priv *priv,
+ struct mlx5_flow_meter_info *fm)
{
#ifdef HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER
rte_spinlock_lock(&fm->sl);
MLX5_ASSERT(fm->ref_cnt);
- if (--fm->ref_cnt == 0) {
- mlx5_glue->destroy_flow_action(fm->mfts->meter_action);
- fm->mfts->meter_action = NULL;
+ if (--fm->ref_cnt == 0 && !priv->sh->meter_aso_en) {
+ mlx5_glue->destroy_flow_action(fm->meter_action);
+ fm->meter_action = NULL;
fm->ingress = 0;
fm->egress = 0;
fm->transfer = 0;