@@ -556,6 +556,7 @@ struct mlx5_priv {
struct mlx5_flows flows; /* RTE Flow rules. */
struct mlx5_flows ctrl_flows; /* Control flow rules. */
void *inter_flows; /* Intermediate resources for flow creation. */
+ void *rss_meta; /* Intermediate rss metadata resources for rss. */
int flow_idx; /* Intermediate device flow index. */
int flow_nested_idx; /* Intermediate device flow index, nested. */
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
@@ -658,13 +658,12 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
* The hash fields that should be used.
*/
uint64_t
-mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow,
+mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_meta *rss_meta,
int tunnel __rte_unused, uint64_t layer_types,
uint64_t hash_fields)
{
- struct rte_flow *flow = dev_flow->flow;
#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
- int rss_request_inner = flow->rss.level >= 2;
+ int rss_request_inner = rss_meta->level >= 2;
/* Check RSS hash level for tunnel. */
if (tunnel && rss_request_inner)
@@ -673,7 +672,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
return 0;
#endif
/* Check if requested layer matches RSS hash fields. */
- if (!(flow->rss.types & layer_types))
+ if (!(rss_meta->types & layer_types))
return 0;
return hash_fields;
}
@@ -4223,6 +4222,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
uint8_t buffer[2048];
} items_tx;
struct rte_flow_expand_rss *buf = &expand_buffer.buf;
+ struct mlx5_flow_rss_meta *rss_meta = priv->rss_meta;
const struct rte_flow_action *p_actions_rx = actions;
uint32_t i;
uint32_t flow_size;
@@ -4263,14 +4263,16 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
MLX5_ASSERT(flow->drv_type > MLX5_FLOW_TYPE_MIN &&
flow->drv_type < MLX5_FLOW_TYPE_MAX);
flow->rss.queue = (void *)(flow + 1);
+ memset(rss_meta, 0, sizeof(*rss_meta));
+ rss_meta->queue = flow->rss.queue;
if (rss) {
/*
* The following information is required by
* mlx5_flow_hashfields_adjust() in advance.
*/
- flow->rss.level = rss->level;
+ rss_meta->level = rss->level;
/* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
- flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
+ rss_meta->types = !rss->types ? ETH_RSS_IP : rss->types;
}
flow->dev_handles = 0;
if (rss && rss->types) {
@@ -4623,9 +4625,17 @@ struct rte_flow *
{
struct mlx5_priv *priv = dev->data->dev_private;
- if (!priv->inter_flows)
+ if (!priv->inter_flows) {
priv->inter_flows = rte_calloc(__func__, MLX5_NUM_MAX_DEV_FLOWS,
- sizeof(struct mlx5_flow), 0);
+ sizeof(struct mlx5_flow) +
+ sizeof(struct mlx5_flow_rss_meta), 0);
+ if (!priv->inter_flows) {
+ DRV_LOG(ERR, "can't allocate intermediate memory.");
+ return;
+ }
+ }
+ priv->rss_meta = &((struct mlx5_flow *)priv->inter_flows)
+ [MLX5_NUM_MAX_DEV_FLOWS];
/* Reset the index. */
priv->flow_idx = 0;
priv->flow_nested_idx = 0;
@@ -482,7 +482,12 @@ struct ibv_spec_header {
uint16_t size;
};
-struct mlx5_flow_rss {
+struct mlx5_flow_rss_queue {
+ uint16_t queue_num; /**< Number of entries in @p queue. */
+ uint16_t (*queue)[]; /**< Destination queues to redirect traffic to. */
+} __rte_packed;
+
+struct mlx5_flow_rss_meta {
uint32_t level;
uint32_t queue_num; /**< Number of entries in @p queue. */
uint64_t types; /**< Specific RSS hash types (see ETH_RSS_*). */
@@ -490,6 +495,7 @@ struct mlx5_flow_rss {
uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */
};
+
/** Device flow handle structure for DV mode only. */
struct mlx5_flow_handle_dv {
/* Flow DV api: */
@@ -749,7 +755,7 @@ struct mlx5_flow_meter_profile {
struct rte_flow {
TAILQ_ENTRY(rte_flow) next; /**< Pointer to the next flow structure. */
enum mlx5_flow_drv_type drv_type; /**< Driver type. */
- struct mlx5_flow_rss rss; /**< RSS context. */
+ struct mlx5_flow_rss_queue rss; /**< RSS queue context. */
uint32_t counter; /**< Holds flow counter. */
struct mlx5_flow_mreg_copy_resource *mreg_copy;
/**< pointer to metadata register copy table resource. */
@@ -842,8 +848,8 @@ uint32_t mlx5_flow_id_release(struct mlx5_flow_id_pool *pool,
int mlx5_flow_group_to_table(const struct rte_flow_attr *attributes,
bool external, uint32_t group, bool fdb_def_rule,
uint32_t *table, struct rte_flow_error *error);
-uint64_t mlx5_flow_hashfields_adjust(struct mlx5_flow *dev_flow, int tunnel,
- uint64_t layer_types,
+uint64_t mlx5_flow_hashfields_adjust(struct mlx5_flow_rss_meta *rss_meta,
+ int tunnel, uint64_t layer_types,
uint64_t hash_fields);
uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
uint32_t subpriority);
@@ -7241,18 +7241,20 @@ struct field_modify_info modify_tcp[] = {
*
* @param[in] dev_flow
* Pointer to the mlx5_flow.
+ * @param[in] rss_meta
+ * Pointer to the mlx5_flow_rss_meta.
*/
static void
-flow_dv_hashfields_set(struct mlx5_flow *dev_flow)
+flow_dv_hashfields_set(struct mlx5_flow *dev_flow,
+ struct mlx5_flow_rss_meta *rss_meta)
{
- struct rte_flow *flow = dev_flow->flow;
uint64_t items = dev_flow->handle->layers;
int rss_inner = 0;
- uint64_t rss_types = rte_eth_rss_hf_refine(flow->rss.types);
+ uint64_t rss_types = rte_eth_rss_hf_refine(rss_meta->types);
dev_flow->hash_fields = 0;
#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
- if (flow->rss.level >= 2) {
+ if (rss_meta->level >= 2) {
dev_flow->hash_fields |= IBV_RX_HASH_INNER;
rss_inner = 1;
}
@@ -7337,6 +7339,8 @@ struct field_modify_info modify_tcp[] = {
struct mlx5_dev_config *dev_conf = &priv->config;
struct rte_flow *flow = dev_flow->flow;
struct mlx5_flow_handle *handle = dev_flow->handle;
+ struct mlx5_flow_rss_meta *rss_meta = (struct mlx5_flow_rss_meta *)
+ priv->rss_meta;
uint64_t item_flags = 0;
uint64_t last_item = 0;
uint64_t action_flags = 0;
@@ -7493,6 +7497,7 @@ struct field_modify_info modify_tcp[] = {
MLX5_ASSERT(flow->rss.queue);
queue = actions->conf;
flow->rss.queue_num = 1;
+ rss_meta->queue_num = flow->rss.queue_num;
(*flow->rss.queue)[0] = queue->index;
action_flags |= MLX5_FLOW_ACTION_QUEUE;
dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
@@ -7504,9 +7509,10 @@ struct field_modify_info modify_tcp[] = {
memcpy((*flow->rss.queue), rss->queue,
rss->queue_num * sizeof(uint16_t));
flow->rss.queue_num = rss->queue_num;
+ rss_meta->queue_num = flow->rss.queue_num;
/* NULL RSS key indicates default RSS key. */
rss_key = !rss->key ? rss_hash_default_key : rss->key;
- memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
+ memcpy(rss_meta->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
/*
* rss->level and rss.types should be set in advance
* when expanding items for RSS.
@@ -7904,7 +7910,7 @@ struct field_modify_info modify_tcp[] = {
case RTE_FLOW_ITEM_TYPE_GRE:
flow_dv_translate_item_gre(match_mask, match_value,
items, tunnel);
- matcher.priority = flow->rss.level >= 2 ?
+ matcher.priority = rss_meta->level >= 2 ?
MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
last_item = MLX5_FLOW_LAYER_GRE;
break;
@@ -7916,14 +7922,14 @@ struct field_modify_info modify_tcp[] = {
case RTE_FLOW_ITEM_TYPE_NVGRE:
flow_dv_translate_item_nvgre(match_mask, match_value,
items, tunnel);
- matcher.priority = flow->rss.level >= 2 ?
+ matcher.priority = rss_meta->level >= 2 ?
MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
last_item = MLX5_FLOW_LAYER_GRE;
break;
case RTE_FLOW_ITEM_TYPE_VXLAN:
flow_dv_translate_item_vxlan(match_mask, match_value,
items, tunnel);
- matcher.priority = flow->rss.level >= 2 ?
+ matcher.priority = rss_meta->level >= 2 ?
MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
last_item = MLX5_FLOW_LAYER_VXLAN;
break;
@@ -7931,21 +7937,21 @@ struct field_modify_info modify_tcp[] = {
flow_dv_translate_item_vxlan_gpe(match_mask,
match_value, items,
tunnel);
- matcher.priority = flow->rss.level >= 2 ?
+ matcher.priority = rss_meta->level >= 2 ?
MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
break;
case RTE_FLOW_ITEM_TYPE_GENEVE:
flow_dv_translate_item_geneve(match_mask, match_value,
items, tunnel);
- matcher.priority = flow->rss.level >= 2 ?
+ matcher.priority = rss_meta->level >= 2 ?
MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
last_item = MLX5_FLOW_LAYER_GENEVE;
break;
case RTE_FLOW_ITEM_TYPE_MPLS:
flow_dv_translate_item_mpls(match_mask, match_value,
items, last_item, tunnel);
- matcher.priority = flow->rss.level >= 2 ?
+ matcher.priority = rss_meta->level >= 2 ?
MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
last_item = MLX5_FLOW_LAYER_MPLS;
break;
@@ -7988,7 +7994,7 @@ struct field_modify_info modify_tcp[] = {
case RTE_FLOW_ITEM_TYPE_GTP:
flow_dv_translate_item_gtp(match_mask, match_value,
items, tunnel);
- matcher.priority = flow->rss.level >= 2 ?
+ matcher.priority = rss_meta->level >= 2 ?
MLX5_PRIORITY_MAP_L2 : MLX5_PRIORITY_MAP_L4;
last_item = MLX5_FLOW_LAYER_GTP;
break;
@@ -8020,7 +8026,7 @@ struct field_modify_info modify_tcp[] = {
*/
handle->layers |= item_flags;
if (action_flags & MLX5_FLOW_ACTION_RSS)
- flow_dv_hashfields_set(dev_flow);
+ flow_dv_hashfields_set(dev_flow, rss_meta);
/* Register matcher. */
matcher.crc = rte_raw_cksum((const void *)matcher.mask.buf,
matcher.mask.size);
@@ -8095,20 +8101,22 @@ struct field_modify_info modify_tcp[] = {
} else if (dh->fate_action == MLX5_FLOW_FATE_QUEUE) {
struct mlx5_hrxq *hrxq;
uint32_t hrxq_idx;
+ struct mlx5_flow_rss_meta *rss_meta =
+ (struct mlx5_flow_rss_meta *)priv->rss_meta;
- MLX5_ASSERT(flow->rss.queue);
- hrxq_idx = mlx5_hrxq_get(dev, flow->rss.key,
+ MLX5_ASSERT(rss_meta->queue);
+ hrxq_idx = mlx5_hrxq_get(dev, rss_meta->key,
MLX5_RSS_HASH_KEY_LEN,
dev_flow->hash_fields,
- (*flow->rss.queue),
- flow->rss.queue_num);
+ (*rss_meta->queue),
+ rss_meta->queue_num);
if (!hrxq_idx) {
hrxq_idx = mlx5_hrxq_new
- (dev, flow->rss.key,
+ (dev, rss_meta->key,
MLX5_RSS_HASH_KEY_LEN,
dev_flow->hash_fields,
- (*flow->rss.queue),
- flow->rss.queue_num,
+ (*rss_meta->queue),
+ rss_meta->queue_num,
!!(dh->layers &
MLX5_FLOW_LAYER_TUNNEL));
}
@@ -941,21 +941,20 @@
* the input is valid and that there is space to insert the requested action
* into the flow.
*
- * @param[in] dev_flow
- * Pointer to mlx5_flow.
+ * @param[in] rss_meta
+ * Pointer to mlx5_flow_rss_meta.
* @param[in] action
* Action configuration.
*/
static void
-flow_verbs_translate_action_queue(struct mlx5_flow *dev_flow,
+flow_verbs_translate_action_queue(struct mlx5_flow_rss_meta *rss_meta,
const struct rte_flow_action *action)
{
const struct rte_flow_action_queue *queue = action->conf;
- struct rte_flow *flow = dev_flow->flow;
- if (flow->rss.queue)
- (*flow->rss.queue)[0] = queue->index;
- flow->rss.queue_num = 1;
+ if (rss_meta->queue)
+ (*rss_meta->queue)[0] = queue->index;
+ rss_meta->queue_num = 1;
}
/**
@@ -963,28 +962,25 @@
* the input is valid and that there is space to insert the requested action
* into the flow.
*
+ * @param[in] rss_meta
+ * Pointer to mlx5_flow_rss_meta.
* @param[in] action
* Action configuration.
- * @param[in, out] action_flags
- * Pointer to the detected actions.
- * @param[in] dev_flow
- * Pointer to mlx5_flow.
*/
static void
-flow_verbs_translate_action_rss(struct mlx5_flow *dev_flow,
+flow_verbs_translate_action_rss(struct mlx5_flow_rss_meta *rss_meta,
const struct rte_flow_action *action)
{
const struct rte_flow_action_rss *rss = action->conf;
const uint8_t *rss_key;
- struct rte_flow *flow = dev_flow->flow;
- if (flow->rss.queue)
- memcpy((*flow->rss.queue), rss->queue,
+ if (rss_meta->queue)
+ memcpy((*rss_meta->queue), rss->queue,
rss->queue_num * sizeof(uint16_t));
- flow->rss.queue_num = rss->queue_num;
+ rss_meta->queue_num = rss->queue_num;
/* NULL RSS key indicates default RSS key. */
rss_key = !rss->key ? rss_hash_default_key : rss->key;
- memcpy(flow->rss.key, rss_key, MLX5_RSS_HASH_KEY_LEN);
+ memcpy(rss_meta->key, rss_key, MLX5_RSS_HASH_KEY_LEN);
/*
* rss->level and rss.types should be set in advance when expanding
* items for RSS.
@@ -1577,6 +1573,8 @@
uint64_t priority = attr->priority;
uint32_t subpriority = 0;
struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_flow_rss_meta *rss_meta = (struct mlx5_flow_rss_meta *)
+ priv->rss_meta;
if (priority == MLX5_FLOW_PRIO_RSVD)
priority = priv->config.flow_prio - 1;
@@ -1602,12 +1600,14 @@
dev_flow->handle->fate_action = MLX5_FLOW_FATE_DROP;
break;
case RTE_FLOW_ACTION_TYPE_QUEUE:
- flow_verbs_translate_action_queue(dev_flow, actions);
+ flow_verbs_translate_action_queue(rss_meta, actions);
action_flags |= MLX5_FLOW_ACTION_QUEUE;
+ dev_flow->flow->rss.queue_num = rss_meta->queue_num;
dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
break;
case RTE_FLOW_ACTION_TYPE_RSS:
- flow_verbs_translate_action_rss(dev_flow, actions);
+ flow_verbs_translate_action_rss(rss_meta, actions);
+ dev_flow->flow->rss.queue_num = rss_meta->queue_num;
action_flags |= MLX5_FLOW_ACTION_RSS;
dev_flow->handle->fate_action = MLX5_FLOW_FATE_QUEUE;
break;
@@ -1655,7 +1655,7 @@
subpriority = MLX5_PRIORITY_MAP_L3;
dev_flow->hash_fields |=
mlx5_flow_hashfields_adjust
- (dev_flow, tunnel,
+ (rss_meta, tunnel,
MLX5_IPV4_LAYER_TYPES,
MLX5_IPV4_IBV_RX_HASH);
item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
@@ -1667,7 +1667,7 @@
subpriority = MLX5_PRIORITY_MAP_L3;
dev_flow->hash_fields |=
mlx5_flow_hashfields_adjust
- (dev_flow, tunnel,
+ (rss_meta, tunnel,
MLX5_IPV6_LAYER_TYPES,
MLX5_IPV6_IBV_RX_HASH);
item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
@@ -1679,7 +1679,7 @@
subpriority = MLX5_PRIORITY_MAP_L4;
dev_flow->hash_fields |=
mlx5_flow_hashfields_adjust
- (dev_flow, tunnel, ETH_RSS_TCP,
+ (rss_meta, tunnel, ETH_RSS_TCP,
(IBV_RX_HASH_SRC_PORT_TCP |
IBV_RX_HASH_DST_PORT_TCP));
item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
@@ -1691,7 +1691,7 @@
subpriority = MLX5_PRIORITY_MAP_L4;
dev_flow->hash_fields |=
mlx5_flow_hashfields_adjust
- (dev_flow, tunnel, ETH_RSS_UDP,
+ (rss_meta, tunnel, ETH_RSS_UDP,
(IBV_RX_HASH_SRC_PORT_UDP |
IBV_RX_HASH_DST_PORT_UDP));
item_flags |= tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
@@ -1848,19 +1848,21 @@
}
} else {
uint32_t hrxq_idx;
+ struct mlx5_flow_rss_meta *rss_meta =
+ (struct mlx5_flow_rss_meta *)priv->rss_meta;
- MLX5_ASSERT(flow->rss.queue);
- hrxq_idx = mlx5_hrxq_get(dev, flow->rss.key,
+ MLX5_ASSERT(rss_meta->queue);
+ hrxq_idx = mlx5_hrxq_get(dev, rss_meta->key,
MLX5_RSS_HASH_KEY_LEN,
dev_flow->hash_fields,
- (*flow->rss.queue),
- flow->rss.queue_num);
+ (*rss_meta->queue),
+ rss_meta->queue_num);
if (!hrxq_idx)
- hrxq_idx = mlx5_hrxq_new(dev, flow->rss.key,
+ hrxq_idx = mlx5_hrxq_new(dev, rss_meta->key,
MLX5_RSS_HASH_KEY_LEN,
dev_flow->hash_fields,
- (*flow->rss.queue),
- flow->rss.queue_num,
+ (*rss_meta->queue),
+ rss_meta->queue_num,
!!(handle->layers &
MLX5_FLOW_LAYER_TUNNEL));
hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],