@@ -1246,6 +1246,8 @@ struct mlx5_flow_id_pool *
*/
mlx5_flow_list_flush(dev, &priv->flows, true);
mlx5_flow_meter_flush(dev, NULL);
+ /* Free the intermediate buffers for flow creation. */
+ mlx5_flow_free_intermediate(dev);
/* Prevent crashes when queues are still in use. */
dev->rx_pkt_burst = removed_rx_burst;
dev->tx_pkt_burst = removed_tx_burst;
@@ -2768,6 +2770,11 @@ struct mlx5_flow_id_pool *
err = ENOTSUP;
goto error;
}
+ /*
+ * Allocate the buffer for flow creating, just once.
+ * The allocation must be done before any flow creating.
+ */
+ mlx5_flow_alloc_intermediate(eth_dev);
/* Query availibility of metadata reg_c's. */
err = mlx5_flow_discover_mreg_c(eth_dev);
if (err < 0) {
@@ -517,6 +517,8 @@ struct mlx5_priv {
struct mlx5_drop drop_queue; /* Flow drop queues. */
struct mlx5_flows flows; /* RTE Flow rules. */
struct mlx5_flows ctrl_flows; /* Control flow rules. */
+ void *inter_flows; /* Intermediate resources for flow creation. */
+ int flow_idx; /* Intermediate device flow index. */
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
@@ -728,6 +730,8 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list);
int mlx5_flow_start_default(struct rte_eth_dev *dev);
void mlx5_flow_stop_default(struct rte_eth_dev *dev);
+void mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev);
+void mlx5_flow_free_intermediate(struct rte_eth_dev *dev);
int mlx5_flow_verify(struct rte_eth_dev *dev);
int mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, uint32_t queue);
int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
@@ -712,17 +712,19 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
*
* @param[in] dev
* Pointer to the Ethernet device structure.
- * @param[in] dev_flow
- * Pointer to device flow structure.
+ * @param[in] flow
+ * Pointer to flow structure.
+ * @param[in] dev_handle
+ * Pointer to device flow handle structure.
*/
static void
-flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
+flow_drv_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow,
+ struct mlx5_flow_handle *dev_handle)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct rte_flow *flow = dev_flow->flow;
- const int mark = !!(dev_flow->handle.act_flags &
+ const int mark = !!(dev_handle->act_flags &
(MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
- const int tunnel = !!(dev_flow->handle.layers & MLX5_FLOW_LAYER_TUNNEL);
+ const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
unsigned int i;
for (i = 0; i != flow->rss.queue_num; ++i) {
@@ -751,7 +753,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
/* Increase the counter matching the flow. */
for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
if ((tunnels_info[j].tunnel &
- dev_flow->handle.layers) ==
+ dev_handle->layers) ==
tunnels_info[j].tunnel) {
rxq_ctrl->flow_tunnels_n[j]++;
break;
@@ -773,10 +775,10 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
static void
flow_rxq_flags_set(struct rte_eth_dev *dev, struct rte_flow *flow)
{
- struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *dev_handle;
- LIST_FOREACH(dev_flow, &flow->dev_flows, next)
- flow_drv_rxq_flags_set(dev, dev_flow);
+ LIST_FOREACH(dev_handle, &flow->dev_handles, next)
+ flow_drv_rxq_flags_set(dev, flow, dev_handle);
}
/**
@@ -785,17 +787,19 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
*
* @param dev
* Pointer to Ethernet device.
- * @param[in] dev_flow
- * Pointer to the device flow.
+ * @param[in] flow
+ * Pointer to flow structure.
+ * @param[in] dev_handle
+ * Pointer to the device flow handle structure.
*/
static void
-flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct mlx5_flow *dev_flow)
+flow_drv_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow,
+ struct mlx5_flow_handle *dev_handle)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct rte_flow *flow = dev_flow->flow;
- const int mark = !!(dev_flow->handle.act_flags &
+ const int mark = !!(dev_handle->act_flags &
(MLX5_FLOW_ACTION_FLAG | MLX5_FLOW_ACTION_MARK));
- const int tunnel = !!(dev_flow->handle.layers & MLX5_FLOW_LAYER_TUNNEL);
+ const int tunnel = !!(dev_handle->layers & MLX5_FLOW_LAYER_TUNNEL);
unsigned int i;
MLX5_ASSERT(dev->data->dev_started);
@@ -820,7 +824,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
/* Decrease the counter matching the flow. */
for (j = 0; j != MLX5_FLOW_TUNNEL; ++j) {
if ((tunnels_info[j].tunnel &
- dev_flow->handle.layers) ==
+ dev_handle->layers) ==
tunnels_info[j].tunnel) {
rxq_ctrl->flow_tunnels_n[j]--;
break;
@@ -843,10 +847,10 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
static void
flow_rxq_flags_trim(struct rte_eth_dev *dev, struct rte_flow *flow)
{
- struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *dev_handle;
- LIST_FOREACH(dev_flow, &flow->dev_flows, next)
- flow_drv_rxq_flags_trim(dev, dev_flow);
+ LIST_FOREACH(dev_handle, &flow->dev_handles, next)
+ flow_drv_rxq_flags_trim(dev, flow, dev_handle);
}
/**
@@ -2309,11 +2313,11 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
flow_mreg_split_qrss_release(struct rte_eth_dev *dev,
struct rte_flow *flow)
{
- struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *dev_handle;
- LIST_FOREACH(dev_flow, &flow->dev_flows, next)
- if (dev_flow->handle.qrss_id)
- flow_qrss_free_id(dev, dev_flow->handle.qrss_id);
+ LIST_FOREACH(dev_handle, &flow->dev_handles, next)
+ if (dev_handle->qrss_id)
+ flow_qrss_free_id(dev, dev_handle->qrss_id);
}
static int
@@ -2329,7 +2333,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
}
static struct mlx5_flow *
-flow_null_prepare(const struct rte_flow_attr *attr __rte_unused,
+flow_null_prepare(struct rte_eth_dev *dev __rte_unused,
+ const struct rte_flow_attr *attr __rte_unused,
const struct rte_flow_item items[] __rte_unused,
const struct rte_flow_action actions[] __rte_unused,
struct rte_flow_error *error)
@@ -2469,6 +2474,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
* setting backward reference to the flow should be done out of this function.
* layers field is not filled either.
*
+ * @param[in] dev
+ * Pointer to the dev structure.
* @param[in] attr
* Pointer to the flow attributes.
* @param[in] items
@@ -2482,7 +2489,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
* Pointer to device flow on success, otherwise NULL and rte_errno is set.
*/
static inline struct mlx5_flow *
-flow_drv_prepare(const struct rte_flow *flow,
+flow_drv_prepare(struct rte_eth_dev *dev,
+ const struct rte_flow *flow,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
@@ -2493,7 +2501,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
MLX5_ASSERT(type > MLX5_FLOW_TYPE_MIN && type < MLX5_FLOW_TYPE_MAX);
fops = flow_get_drv_ops(type);
- return fops->prepare(attr, items, actions, error);
+ return fops->prepare(dev, attr, items, actions, error);
}
/**
@@ -2701,17 +2709,17 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
* help to do the optimization work for source code.
* If no decap actions, use the layers directly.
*/
- if (!(dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DECAP))
- return dev_flow->handle.layers;
+ if (!(dev_flow->handle->act_flags & MLX5_FLOW_ACTION_DECAP))
+ return dev_flow->handle->layers;
/* Convert L3 layers with decap action. */
- if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
+ if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV4)
layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV4;
- else if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
+ else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L3_IPV6)
layers |= MLX5_FLOW_LAYER_OUTER_L3_IPV6;
/* Convert L4 layers with decap action. */
- if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
+ if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_TCP)
layers |= MLX5_FLOW_LAYER_OUTER_L4_TCP;
- else if (dev_flow->handle.layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
+ else if (dev_flow->handle->layers & MLX5_FLOW_LAYER_INNER_L4_UDP)
layers |= MLX5_FLOW_LAYER_OUTER_L4_UDP;
return layers;
}
@@ -3412,7 +3420,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
* The last stage of splitting chain, just creates the subflow
* without any modification.
*
- * @param dev
+ * @param[in] dev
* Pointer to Ethernet device.
* @param[in] flow
* Parent flow structure pointer.
@@ -3445,19 +3453,19 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
{
struct mlx5_flow *dev_flow;
- dev_flow = flow_drv_prepare(flow, attr, items, actions, error);
+ dev_flow = flow_drv_prepare(dev, flow, attr, items, actions, error);
if (!dev_flow)
return -rte_errno;
dev_flow->flow = flow;
dev_flow->external = external;
/* Subflow object was created, we must include one in the list. */
- LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
+ LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next);
/*
* If dev_flow is as one of the suffix flow, some actions in suffix
* flow may need some user defined item layer flags.
*/
if (prefix_layers)
- dev_flow->handle.layers = prefix_layers;
+ dev_flow->handle->layers = prefix_layers;
if (sub_flow)
*sub_flow = dev_flow;
return flow_drv_translate(dev, dev_flow, attr, items, actions, error);
@@ -3972,7 +3980,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
* reallocation becomes possible (for example, for
* other flows in other threads).
*/
- dev_flow->handle.qrss_id = qrss_id;
+ dev_flow->handle->qrss_id = qrss_id;
ret = mlx5_flow_get_reg_id(dev, MLX5_COPY_MARK, 0,
error);
if (ret < 0)
@@ -4085,7 +4093,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
ret = -rte_errno;
goto exit;
}
- dev_flow->handle.mtr_flow_id = mtr_tag_id;
+ dev_flow->handle->mtr_flow_id = mtr_tag_id;
/* Setting the sfx group atrr. */
sfx_attr.group = sfx_attr.transfer ?
(MLX5_FLOW_TABLE_LEVEL_SUFFIX - 1) :
@@ -4256,7 +4264,7 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
/* RSS type 0 indicates default RSS type (ETH_RSS_IP). */
flow->rss.types = !rss->types ? ETH_RSS_IP : rss->types;
}
- LIST_INIT(&flow->dev_flows);
+ LIST_INIT(&flow->dev_handles);
if (rss && rss->types) {
unsigned int graph_root;
@@ -4271,6 +4279,8 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
buf->entries = 1;
buf->entry[0].pattern = (void *)(uintptr_t)items;
}
+ /* Reset device flow index to 0. */
+ priv->flow_idx = 0;
for (i = 0; i < buf->entries; ++i) {
/*
* The splitter may create multiple dev_flows,
@@ -4289,13 +4299,13 @@ uint32_t mlx5_flow_adjust_priority(struct rte_eth_dev *dev, int32_t priority,
attr_tx.group = MLX5_HAIRPIN_TX_TABLE;
attr_tx.ingress = 0;
attr_tx.egress = 1;
- dev_flow = flow_drv_prepare(flow, &attr_tx, items_tx.items,
+ dev_flow = flow_drv_prepare(dev, flow, &attr_tx, items_tx.items,
actions_hairpin_tx.actions, error);
if (!dev_flow)
goto error;
dev_flow->flow = flow;
dev_flow->external = 0;
- LIST_INSERT_HEAD(&flow->dev_flows, dev_flow, next);
+ LIST_INSERT_HEAD(&flow->dev_handles, dev_flow->handle, next);
ret = flow_drv_translate(dev, dev_flow, &attr_tx,
items_tx.items,
actions_hairpin_tx.actions, error);
@@ -4543,8 +4553,6 @@ struct rte_flow *
*
* @param dev
* Pointer to Ethernet device.
- * @param list
- * Pointer to a TAILQ flow list.
*/
void
mlx5_flow_stop_default(struct rte_eth_dev *dev)
@@ -4570,6 +4578,37 @@ struct rte_flow *
}
/**
+ * Allocate intermediate resources for flow creation.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ if (!priv->inter_flows)
+ priv->inter_flows = rte_calloc(__func__, MLX5_NUM_MAX_DEV_FLOWS,
+ sizeof(struct mlx5_flow), 0);
+}
+
+/**
+ * Free intermediate resources for flows.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_flow_free_intermediate(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ rte_free(priv->inter_flows);
+ priv->inter_flows = NULL;
+}
+
+/**
* Verify the flow list is empty
*
* @param dev
@@ -500,6 +500,8 @@ struct mlx5_flow_handle_dv {
/** Device flow handle structure: used both for creating & destroying. */
struct mlx5_flow_handle {
+ LIST_ENTRY(mlx5_flow_handle) next;
+ /**< Pointer to next device flow handle. */
uint64_t layers;
/**< Bit-fields of present layers, see MLX5_FLOW_LAYER_*. */
uint64_t act_flags;
@@ -517,6 +519,18 @@ struct mlx5_flow_handle {
};
/*
+ * Size for Verbs device flow handle structure only. Do not use the DV only
+ * structure in Verbs. No DV flows attributes will be accessed.
+ * Macro offsetof() could also be used here.
+ */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+#define MLX5_FLOW_HANDLE_VERBS_SIZE \
+ (sizeof(struct mlx5_flow_handle) - sizeof(struct mlx5_flow_handle_dv))
+#else
+#define MLX5_FLOW_HANDLE_VERBS_SIZE (sizeof(struct mlx5_flow_handle))
+#endif
+
+/*
* Max number of actions per DV flow.
* See CREATE_FLOW_MAX_FLOW_ACTIONS_SUPPORTED
* in rdma-core file providers/mlx5/verbs.c.
@@ -524,7 +538,7 @@ struct mlx5_flow_handle {
#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8
/** Device flow structure only for DV flow creation. */
-struct mlx5_flow_resource_dv {
+struct mlx5_flow_dv_workspace {
uint32_t group; /**< The group index. */
uint8_t transfer; /**< 1 if the flow is E-Switch flow. */
int actions_n; /**< number of actions. */
@@ -533,27 +547,79 @@ struct mlx5_flow_resource_dv {
/**< Holds the value that the packet is compared to. */
};
+/*
+ * Maximal Verbs flow specifications & actions size.
+ * Some elements are mutually exclusive, but enough space should be allocated.
+ * Tunnel cases: 1. Max 2 Ethernet + IP(v6 len > v4 len) + TCP/UDP headers.
+ * 2. One tunnel header (exception: GRE + MPLS),
+ * SPEC length: GRE == tunnel.
+ * Actions: 1. 1 Mark OR Flag.
+ * 2. 1 Drop (if any).
+ * 3. No limitation for counters, but it makes no sense to support too
+ * many counters in a single device flow.
+ */
+#ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT
+#define MLX5_VERBS_MAX_SPEC_SIZE \
+ ( \
+ (2 * (sizeof(struct ibv_flow_spec_eth) + \
+ sizeof(struct ibv_flow_spec_ipv6) + \
+ sizeof(struct ibv_flow_spec_tcp_udp)) + \
+ sizeof(struct ibv_flow_spec_gre) + \
+ sizeof(struct ibv_flow_spec_mpls)) \
+ )
+#else
+#define MLX5_VERBS_MAX_SPEC_SIZE \
+ ( \
+ (2 * (sizeof(struct ibv_flow_spec_eth) + \
+ sizeof(struct ibv_flow_spec_ipv6) + \
+ sizeof(struct ibv_flow_spec_tcp_udp)) + \
+ sizeof(struct ibv_flow_spec_tunnel)) \
+ )
+#endif
+
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) || \
+ defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+#define MLX5_VERBS_MAX_ACT_SIZE \
+ ( \
+ sizeof(struct ibv_flow_spec_action_tag) + \
+ sizeof(struct ibv_flow_spec_action_drop) + \
+ sizeof(struct ibv_flow_spec_counter_action) * 4 \
+ )
+#else
+#define MLX5_VERBS_MAX_ACT_SIZE \
+ ( \
+ sizeof(struct ibv_flow_spec_action_tag) + \
+ sizeof(struct ibv_flow_spec_action_drop) \
+ )
+#endif
+
+#define MLX5_VERBS_MAX_SPEC_ACT_SIZE \
+ (MLX5_VERBS_MAX_SPEC_SIZE + MLX5_VERBS_MAX_ACT_SIZE)
+
/** Device flow structure only for Verbs flow creation. */
-struct mlx5_flow_resource_verbs {
+struct mlx5_flow_verbs_workspace {
unsigned int size; /**< Size of the attribute. */
- struct ibv_flow_attr *attr; /**< Pointer to the Specification buffer. */
- uint8_t *specs; /**< Pointer to the specifications. */
+ struct ibv_flow_attr attr; /**< Verbs flow attribute buffer. */
+ uint8_t specs[MLX5_VERBS_MAX_SPEC_ACT_SIZE];
+ /**< Specifications & actions buffer of verbs flow. */
};
+/** Maximal number of device sub-flows supported. */
+#define MLX5_NUM_MAX_DEV_FLOWS 32
+
/** Device flow structure. */
struct mlx5_flow {
- LIST_ENTRY(mlx5_flow) next; /**< Pointer to next device flow. */
struct rte_flow *flow; /**< Pointer to the main flow. */
uint64_t hash_fields; /**< Verbs hash Rx queue hash fields. */
bool external; /**< true if the flow is created external to PMD. */
uint8_t ingress; /**< 1 if the flow is ingress. */
union {
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
- struct mlx5_flow_resource_dv dv;
+ struct mlx5_flow_dv_workspace dv;
#endif
- struct mlx5_flow_resource_verbs verbs;
+ struct mlx5_flow_verbs_workspace verbs;
};
- struct mlx5_flow_handle handle;
+ struct mlx5_flow_handle *handle;
};
/* Flow meter state. */
@@ -667,8 +733,8 @@ struct rte_flow {
struct mlx5_flow_mreg_copy_resource *mreg_copy;
/**< pointer to metadata register copy table resource. */
struct mlx5_flow_meter *meter; /**< Holds flow meter. */
- LIST_HEAD(dev_flows, mlx5_flow) dev_flows;
- /**< Device flows that are part of the flow. */
+ LIST_HEAD(dev_handles, mlx5_flow_handle) dev_handles;
+ /**< Device flow handles that are part of the flow. */
struct mlx5_fdir *fdir; /**< Pointer to associated FDIR if any. */
uint32_t hairpin_flow_id; /**< The flow id used for hairpin. */
uint32_t copy_applied:1; /**< The MARK copy Flow os applied. */
@@ -681,7 +747,8 @@ typedef int (*mlx5_flow_validate_t)(struct rte_eth_dev *dev,
bool external,
struct rte_flow_error *error);
typedef struct mlx5_flow *(*mlx5_flow_prepare_t)
- (const struct rte_flow_attr *attr, const struct rte_flow_item items[],
+ (struct rte_eth_dev *dev, const struct rte_flow_attr *attr,
+ const struct rte_flow_item items[],
const struct rte_flow_action actions[], struct rte_flow_error *error);
typedef int (*mlx5_flow_translate_t)(struct rte_eth_dev *dev,
struct mlx5_flow *dev_flow,
@@ -92,7 +92,7 @@
flow_dv_attr_init(const struct rte_flow_item *item, union flow_dv_attr *attr,
struct mlx5_flow *dev_flow, bool tunnel_decap)
{
- uint64_t layers = dev_flow->handle.layers;
+ uint64_t layers = dev_flow->handle->layers;
/*
* If layers is already initialized, it means this dev_flow is the
@@ -2399,7 +2399,7 @@ struct field_modify_info modify_tcp[] = {
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
rte_atomic32_inc(&cache_resource->refcnt);
- dev_flow->handle.dvh.encap_decap = cache_resource;
+ dev_flow->handle->dvh.encap_decap = cache_resource;
return 0;
}
}
@@ -2425,7 +2425,7 @@ struct field_modify_info modify_tcp[] = {
rte_atomic32_init(&cache_resource->refcnt);
rte_atomic32_inc(&cache_resource->refcnt);
LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
- dev_flow->handle.dvh.encap_decap = cache_resource;
+ dev_flow->handle->dvh.encap_decap = cache_resource;
DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
@@ -2476,7 +2476,7 @@ struct field_modify_info modify_tcp[] = {
(void *)&tbl_data->jump, cnt);
}
rte_atomic32_inc(&tbl_data->jump.refcnt);
- dev_flow->handle.dvh.jump = &tbl_data->jump;
+ dev_flow->handle->dvh.jump = &tbl_data->jump;
return 0;
}
@@ -2514,7 +2514,7 @@ struct field_modify_info modify_tcp[] = {
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
rte_atomic32_inc(&cache_resource->refcnt);
- dev_flow->handle.dvh.port_id_action = cache_resource;
+ dev_flow->handle->dvh.port_id_action = cache_resource;
return 0;
}
}
@@ -2542,7 +2542,7 @@ struct field_modify_info modify_tcp[] = {
rte_atomic32_init(&cache_resource->refcnt);
rte_atomic32_inc(&cache_resource->refcnt);
LIST_INSERT_HEAD(&sh->port_id_action_list, cache_resource, next);
- dev_flow->handle.dvh.port_id_action = cache_resource;
+ dev_flow->handle->dvh.port_id_action = cache_resource;
DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
@@ -2585,7 +2585,7 @@ struct field_modify_info modify_tcp[] = {
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
rte_atomic32_inc(&cache_resource->refcnt);
- dev_flow->handle.dvh.push_vlan_res = cache_resource;
+ dev_flow->handle->dvh.push_vlan_res = cache_resource;
return 0;
}
}
@@ -2614,7 +2614,7 @@ struct field_modify_info modify_tcp[] = {
rte_atomic32_init(&cache_resource->refcnt);
rte_atomic32_inc(&cache_resource->refcnt);
LIST_INSERT_HEAD(&sh->push_vlan_action_list, cache_resource, next);
- dev_flow->handle.dvh.push_vlan_res = cache_resource;
+ dev_flow->handle->dvh.push_vlan_res = cache_resource;
DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
@@ -3727,7 +3727,7 @@ struct field_modify_info modify_tcp[] = {
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
rte_atomic32_inc(&cache_resource->refcnt);
- dev_flow->handle.dvh.modify_hdr = cache_resource;
+ dev_flow->handle->dvh.modify_hdr = cache_resource;
return 0;
}
}
@@ -3754,7 +3754,7 @@ struct field_modify_info modify_tcp[] = {
rte_atomic32_init(&cache_resource->refcnt);
rte_atomic32_inc(&cache_resource->refcnt);
LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
- dev_flow->handle.dvh.modify_hdr = cache_resource;
+ dev_flow->handle->dvh.modify_hdr = cache_resource;
DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
@@ -5207,6 +5207,8 @@ struct field_modify_info modify_tcp[] = {
* Internal preparation function. Allocates the DV flow size,
* this size is constant.
*
+ * @param[in] dev
+ * Pointer to the rte_eth_dev structure.
* @param[in] attr
* Pointer to the flow attributes.
* @param[in] items
@@ -5221,22 +5223,41 @@ struct field_modify_info modify_tcp[] = {
* otherwise NULL and rte_errno is set.
*/
static struct mlx5_flow *
-flow_dv_prepare(const struct rte_flow_attr *attr __rte_unused,
+flow_dv_prepare(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr __rte_unused,
const struct rte_flow_item items[] __rte_unused,
const struct rte_flow_action actions[] __rte_unused,
struct rte_flow_error *error)
{
- size_t size = sizeof(struct mlx5_flow);
+ size_t size = sizeof(struct mlx5_flow_handle);
struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *dev_handle;
+ struct mlx5_priv *priv = dev->data->dev_private;
- dev_flow = rte_calloc(__func__, 1, size, 0);
- if (!dev_flow) {
+ /* In case of corrupting the memory. */
+ if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
+ rte_flow_error_set(error, ENOSPC,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "not free temporary device flow");
+ return NULL;
+ }
+ dev_handle = rte_calloc(__func__, 1, size, 0);
+ if (!dev_handle) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- "not enough memory to create flow");
+ "not enough memory to create flow handle");
return NULL;
}
+ /* No multi-thread supporting. */
+ dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
+ dev_flow->handle = dev_handle;
dev_flow->dv.value.size = MLX5_ST_SZ_BYTES(fte_match_param);
+ /*
+ * The matching value needs to be cleared to 0 before using. In the
+ * past, it will be automaticlly cleared when using rte_*alloc
+ * API. The time consumption will be almost the same as before.
+ */
+ memset(dev_flow->dv.value.buf, 0, MLX5_ST_SZ_BYTES(fte_match_param));
dev_flow->ingress = attr->ingress;
dev_flow->dv.transfer = attr->transfer;
return dev_flow;
@@ -5394,7 +5415,7 @@ struct field_modify_info modify_tcp[] = {
* This is workaround, masks are not supported,
* and pre-validated.
*/
- dev_flow->handle.vf_vlan.tag =
+ dev_flow->handle->vf_vlan.tag =
rte_be_to_cpu_16(vlan_v->tci) & 0x0fff;
}
tci_m = rte_be_to_cpu_16(vlan_m->tci);
@@ -6895,7 +6916,7 @@ struct field_modify_info modify_tcp[] = {
(void *)cache_matcher,
rte_atomic32_read(&cache_matcher->refcnt));
rte_atomic32_inc(&cache_matcher->refcnt);
- dev_flow->handle.dvh.matcher = cache_matcher;
+ dev_flow->handle->dvh.matcher = cache_matcher;
/* old matcher should not make the table ref++. */
flow_dv_tbl_resource_release(dev, tbl);
return 0;
@@ -6932,7 +6953,7 @@ struct field_modify_info modify_tcp[] = {
/* only matcher ref++, table ref++ already done above in get API. */
rte_atomic32_inc(&cache_matcher->refcnt);
LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
- dev_flow->handle.dvh.matcher = cache_matcher;
+ dev_flow->handle->dvh.matcher = cache_matcher;
DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
key->domain ? "FDB" : "NIC", key->table_id,
cache_matcher->priority,
@@ -6974,7 +6995,7 @@ struct field_modify_info modify_tcp[] = {
cache_resource = container_of
(entry, struct mlx5_flow_dv_tag_resource, entry);
rte_atomic32_inc(&cache_resource->refcnt);
- dev_flow->handle.dvh.tag_resource = cache_resource;
+ dev_flow->handle->dvh.tag_resource = cache_resource;
DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
@@ -7003,7 +7024,7 @@ struct field_modify_info modify_tcp[] = {
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "cannot insert tag");
}
- dev_flow->handle.dvh.tag_resource = cache_resource;
+ dev_flow->handle->dvh.tag_resource = cache_resource;
DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
(void *)cache_resource,
rte_atomic32_read(&cache_resource->refcnt));
@@ -7148,7 +7169,7 @@ struct field_modify_info modify_tcp[] = {
flow_dv_hashfields_set(struct mlx5_flow *dev_flow)
{
struct rte_flow *flow = dev_flow->flow;
- uint64_t items = dev_flow->handle.layers;
+ uint64_t items = dev_flow->handle->layers;
int rss_inner = 0;
uint64_t rss_types = rte_eth_rss_hf_refine(flow->rss.types);
@@ -7238,6 +7259,7 @@ struct field_modify_info modify_tcp[] = {
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_dev_config *dev_conf = &priv->config;
struct rte_flow *flow = dev_flow->flow;
+ struct mlx5_flow_handle *handle = dev_flow->handle;
uint64_t item_flags = 0;
uint64_t last_item = 0;
uint64_t action_flags = 0;
@@ -7306,7 +7328,7 @@ struct field_modify_info modify_tcp[] = {
(dev, &port_id_resource, dev_flow, error))
return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.port_id_action->action;
+ handle->dvh.port_id_action->action;
action_flags |= MLX5_FLOW_ACTION_PORT_ID;
break;
case RTE_FLOW_ACTION_TYPE_FLAG:
@@ -7324,12 +7346,17 @@ struct field_modify_info modify_tcp[] = {
break;
}
tag_be = mlx5_flow_mark_set(MLX5_FLOW_MARK_DEFAULT);
- if (!dev_flow->handle.dvh.tag_resource)
- if (flow_dv_tag_resource_register
- (dev, tag_be, dev_flow, error))
- return -rte_errno;
+ /*
+ * Only one FLAG or MARK is supported per device flow
+ * right now. So the pointer to the tag resource must be
+ * zero before the register process.
+ */
+ MLX5_ASSERT(!handle->dvh.tag_resource);
+ if (flow_dv_tag_resource_register(dev, tag_be,
+ dev_flow, error))
+ return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.tag_resource->action;
+ handle->dvh.tag_resource->action;
break;
case RTE_FLOW_ACTION_TYPE_MARK:
action_flags |= MLX5_FLOW_ACTION_MARK;
@@ -7351,12 +7378,12 @@ struct field_modify_info modify_tcp[] = {
tag_be = mlx5_flow_mark_set
(((const struct rte_flow_action_mark *)
(actions->conf))->id);
- if (!dev_flow->handle.dvh.tag_resource)
- if (flow_dv_tag_resource_register
- (dev, tag_be, dev_flow, error))
- return -rte_errno;
+ MLX5_ASSERT(!handle->dvh.tag_resource);
+ if (flow_dv_tag_resource_register(dev, tag_be,
+ dev_flow, error))
+ return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.tag_resource->action;
+ handle->dvh.tag_resource->action;
break;
case RTE_FLOW_ACTION_TYPE_SET_META:
if (flow_dv_convert_action_set_meta
@@ -7454,7 +7481,7 @@ struct field_modify_info modify_tcp[] = {
(dev, attr, &vlan, dev_flow, error))
return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.push_vlan_res->action;
+ handle->dvh.push_vlan_res->action;
action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
break;
case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP:
@@ -7481,7 +7508,7 @@ struct field_modify_info modify_tcp[] = {
error))
return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.encap_decap->verbs_action;
+ handle->dvh.encap_decap->verbs_action;
action_flags |= MLX5_FLOW_ACTION_ENCAP;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
@@ -7491,7 +7518,7 @@ struct field_modify_info modify_tcp[] = {
error))
return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.encap_decap->verbs_action;
+ handle->dvh.encap_decap->verbs_action;
action_flags |= MLX5_FLOW_ACTION_DECAP;
break;
case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
@@ -7501,7 +7528,7 @@ struct field_modify_info modify_tcp[] = {
(dev, actions, dev_flow, attr, error))
return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.encap_decap->verbs_action;
+ handle->dvh.encap_decap->verbs_action;
} else {
/* Handle encap without preceding decap. */
if (flow_dv_create_action_l2_encap
@@ -7509,7 +7536,7 @@ struct field_modify_info modify_tcp[] = {
error))
return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.encap_decap->verbs_action;
+ handle->dvh.encap_decap->verbs_action;
}
action_flags |= MLX5_FLOW_ACTION_ENCAP;
break;
@@ -7521,7 +7548,7 @@ struct field_modify_info modify_tcp[] = {
(dev, dev_flow, attr->transfer, error))
return -rte_errno;
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.encap_decap->verbs_action;
+ handle->dvh.encap_decap->verbs_action;
}
/* If decap is followed by encap, handle it at encap. */
action_flags |= MLX5_FLOW_ACTION_DECAP;
@@ -7553,7 +7580,7 @@ struct field_modify_info modify_tcp[] = {
"cannot create jump action.");
}
dev_flow->dv.actions[actions_n++] =
- dev_flow->handle.dvh.jump->action;
+ handle->dvh.jump->action;
action_flags |= MLX5_FLOW_ACTION_JUMP;
break;
case RTE_FLOW_ACTION_TYPE_SET_MAC_SRC:
@@ -7686,7 +7713,7 @@ struct field_modify_info modify_tcp[] = {
(dev, mhdr_res, dev_flow, error))
return -rte_errno;
dev_flow->dv.actions[modify_action_position] =
- dev_flow->handle.dvh.modify_hdr->verbs_action;
+ handle->dvh.modify_hdr->verbs_action;
}
break;
default:
@@ -7697,7 +7724,7 @@ struct field_modify_info modify_tcp[] = {
modify_action_position = actions_n++;
}
dev_flow->dv.actions_n = actions_n;
- dev_flow->handle.act_flags = action_flags;
+ handle->act_flags = action_flags;
for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
int item_type = items->type;
@@ -7902,7 +7929,7 @@ struct field_modify_info modify_tcp[] = {
* Layers may be already initialized from prefix flow if this dev_flow
* is the suffix flow.
*/
- dev_flow->handle.layers |= item_flags;
+ handle->layers |= item_flags;
if (action_flags & MLX5_FLOW_ACTION_RSS)
flow_dv_hashfields_set(dev_flow);
/* Register matcher. */
@@ -7937,19 +7964,21 @@ struct field_modify_info modify_tcp[] = {
__flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
struct rte_flow_error *error)
{
- struct mlx5_flow_resource_dv *dv;
+ struct mlx5_flow_dv_workspace *dv;
struct mlx5_flow_handle *dh;
struct mlx5_flow_handle_dv *dv_h;
struct mlx5_flow *dev_flow;
struct mlx5_priv *priv = dev->data->dev_private;
int n;
int err;
+ int idx;
- LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
- dh = &dev_flow->handle;
+ for (idx = priv->flow_idx - 1; idx >= 0; idx--) {
+ dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
dv = &dev_flow->dv;
- n = dv->actions_n;
+ dh = dev_flow->handle;
dv_h = &dh->dvh;
+ n = dv->actions_n;
if (dh->act_flags & MLX5_FLOW_ACTION_DROP) {
if (dv->transfer) {
dv->actions[n++] = priv->sh->esw_drop_action;
@@ -7982,7 +8011,7 @@ struct field_modify_info modify_tcp[] = {
dev_flow->hash_fields,
(*flow->rss.queue),
flow->rss.queue_num,
- !!(dev_flow->handle.layers &
+ !!(dh->layers &
MLX5_FLOW_LAYER_TUNNEL));
}
if (!hrxq) {
@@ -8020,17 +8049,16 @@ struct field_modify_info modify_tcp[] = {
return 0;
error:
err = rte_errno; /* Save rte_errno before cleanup. */
- LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
- struct mlx5_flow_handle *dh_tmp = &dev_flow->handle;
- if (dh_tmp->hrxq) {
- if (dh_tmp->act_flags & MLX5_FLOW_ACTION_DROP)
+ LIST_FOREACH(dh, &flow->dev_handles, next) {
+ if (dh->hrxq) {
+ if (dh->act_flags & MLX5_FLOW_ACTION_DROP)
mlx5_hrxq_drop_release(dev);
else
- mlx5_hrxq_release(dev, dh_tmp->hrxq);
- dh_tmp->hrxq = NULL;
+ mlx5_hrxq_release(dev, dh->hrxq);
+ dh->hrxq = NULL;
}
- if (dh_tmp->vf_vlan.tag && dh_tmp->vf_vlan.created)
- mlx5_vlan_vmwa_release(dev, &dh_tmp->vf_vlan);
+ if (dh->vf_vlan.tag && dh->vf_vlan.created)
+ mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
}
rte_errno = err; /* Restore rte_errno. */
return -rte_errno;
@@ -8041,17 +8069,17 @@ struct field_modify_info modify_tcp[] = {
*
* @param dev
* Pointer to Ethernet device.
- * @param flow
- * Pointer to mlx5_flow.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
*
* @return
* 1 while a reference on it exists, 0 when freed.
*/
static int
flow_dv_matcher_release(struct rte_eth_dev *dev,
- struct mlx5_flow *flow)
+ struct mlx5_flow_handle *handle)
{
- struct mlx5_flow_dv_matcher *matcher = flow->handle.dvh.matcher;
+ struct mlx5_flow_dv_matcher *matcher = handle->dvh.matcher;
MLX5_ASSERT(matcher->matcher_object);
DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
@@ -8074,17 +8102,17 @@ struct field_modify_info modify_tcp[] = {
/**
* Release an encap/decap resource.
*
- * @param flow
- * Pointer to mlx5_flow.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
*
* @return
* 1 while a reference on it exists, 0 when freed.
*/
static int
-flow_dv_encap_decap_resource_release(struct mlx5_flow *flow)
+flow_dv_encap_decap_resource_release(struct mlx5_flow_handle *handle)
{
struct mlx5_flow_dv_encap_decap_resource *cache_resource =
- flow->handle.dvh.encap_decap;
+ handle->dvh.encap_decap;
MLX5_ASSERT(cache_resource->verbs_action);
DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
@@ -8107,18 +8135,18 @@ struct field_modify_info modify_tcp[] = {
*
* @param dev
* Pointer to Ethernet device.
- * @param flow
- * Pointer to mlx5_flow.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
*
* @return
* 1 while a reference on it exists, 0 when freed.
*/
static int
flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
- struct mlx5_flow *flow)
+ struct mlx5_flow_handle *handle)
{
struct mlx5_flow_dv_jump_tbl_resource *cache_resource =
- flow->handle.dvh.jump;
+ handle->dvh.jump;
struct mlx5_flow_tbl_data_entry *tbl_data =
container_of(cache_resource,
struct mlx5_flow_tbl_data_entry, jump);
@@ -8142,17 +8170,17 @@ struct field_modify_info modify_tcp[] = {
/**
* Release a modify-header resource.
*
- * @param flow
- * Pointer to mlx5_flow.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
*
* @return
* 1 while a reference on it exists, 0 when freed.
*/
static int
-flow_dv_modify_hdr_resource_release(struct mlx5_flow *flow)
+flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
{
struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
- flow->handle.dvh.modify_hdr;
+ handle->dvh.modify_hdr;
MLX5_ASSERT(cache_resource->verbs_action);
DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
@@ -8173,17 +8201,17 @@ struct field_modify_info modify_tcp[] = {
/**
* Release port ID action resource.
*
- * @param flow
- * Pointer to mlx5_flow.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
*
* @return
* 1 while a reference on it exists, 0 when freed.
*/
static int
-flow_dv_port_id_action_resource_release(struct mlx5_flow *flow)
+flow_dv_port_id_action_resource_release(struct mlx5_flow_handle *handle)
{
struct mlx5_flow_dv_port_id_action_resource *cache_resource =
- flow->handle.dvh.port_id_action;
+ handle->dvh.port_id_action;
MLX5_ASSERT(cache_resource->action);
DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
@@ -8204,17 +8232,17 @@ struct field_modify_info modify_tcp[] = {
/**
* Release push vlan action resource.
*
- * @param flow
- * Pointer to mlx5_flow.
+ * @param handle
+ * Pointer to mlx5_flow_handle.
*
* @return
* 1 while a reference on it exists, 0 when freed.
*/
static int
-flow_dv_push_vlan_action_resource_release(struct mlx5_flow *flow)
+flow_dv_push_vlan_action_resource_release(struct mlx5_flow_handle *handle)
{
struct mlx5_flow_dv_push_vlan_action_resource *cache_resource =
- flow->handle.dvh.push_vlan_res;
+ handle->dvh.push_vlan_res;
MLX5_ASSERT(cache_resource->action);
DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
@@ -8245,18 +8273,16 @@ struct field_modify_info modify_tcp[] = {
__flow_dv_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
{
struct mlx5_flow_handle *dh;
- struct mlx5_flow *dev_flow;
if (!flow)
return;
- LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
- dh = &dev_flow->handle;
+ LIST_FOREACH(dh, &flow->dev_handles, next) {
if (dh->ib_flow) {
claim_zero(mlx5_glue->dv_destroy_flow(dh->ib_flow));
dh->ib_flow = NULL;
}
if (dh->hrxq) {
- if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP)
+ if (dh->act_flags & MLX5_FLOW_ACTION_DROP)
mlx5_hrxq_drop_release(dev);
else
mlx5_hrxq_release(dev, dh->hrxq);
@@ -8279,7 +8305,7 @@ struct field_modify_info modify_tcp[] = {
static void
__flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
{
- struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *dev_handle;
if (!flow)
return;
@@ -8292,25 +8318,25 @@ struct field_modify_info modify_tcp[] = {
mlx5_flow_meter_detach(flow->meter);
flow->meter = NULL;
}
- while (!LIST_EMPTY(&flow->dev_flows)) {
- dev_flow = LIST_FIRST(&flow->dev_flows);
- LIST_REMOVE(dev_flow, next);
- if (dev_flow->handle.dvh.matcher)
- flow_dv_matcher_release(dev, dev_flow);
- if (dev_flow->handle.dvh.encap_decap)
- flow_dv_encap_decap_resource_release(dev_flow);
- if (dev_flow->handle.dvh.modify_hdr)
- flow_dv_modify_hdr_resource_release(dev_flow);
- if (dev_flow->handle.dvh.jump)
- flow_dv_jump_tbl_resource_release(dev, dev_flow);
- if (dev_flow->handle.dvh.port_id_action)
- flow_dv_port_id_action_resource_release(dev_flow);
- if (dev_flow->handle.dvh.push_vlan_res)
- flow_dv_push_vlan_action_resource_release(dev_flow);
- if (dev_flow->handle.dvh.tag_resource)
+ while (!LIST_EMPTY(&flow->dev_handles)) {
+ dev_handle = LIST_FIRST(&flow->dev_handles);
+ LIST_REMOVE(dev_handle, next);
+ if (dev_handle->dvh.matcher)
+ flow_dv_matcher_release(dev, dev_handle);
+ if (dev_handle->dvh.encap_decap)
+ flow_dv_encap_decap_resource_release(dev_handle);
+ if (dev_handle->dvh.modify_hdr)
+ flow_dv_modify_hdr_resource_release(dev_handle);
+ if (dev_handle->dvh.jump)
+ flow_dv_jump_tbl_resource_release(dev, dev_handle);
+ if (dev_handle->dvh.port_id_action)
+ flow_dv_port_id_action_resource_release(dev_handle);
+ if (dev_handle->dvh.push_vlan_res)
+ flow_dv_push_vlan_action_resource_release(dev_handle);
+ if (dev_handle->dvh.tag_resource)
flow_dv_tag_release(dev,
- dev_flow->handle.dvh.tag_resource);
- rte_free(dev_flow);
+ dev_handle->dvh.tag_resource);
+ rte_free(dev_handle);
}
}
@@ -253,7 +253,7 @@
* Size in bytes of the specification to copy.
*/
static void
-flow_verbs_spec_add(struct mlx5_flow_resource_verbs *verbs,
+flow_verbs_spec_add(struct mlx5_flow_verbs_workspace *verbs,
void *src, unsigned int size)
{
void *dst;
@@ -263,7 +263,7 @@
MLX5_ASSERT(verbs->specs);
dst = (void *)(verbs->specs + verbs->size);
memcpy(dst, src, size);
- ++verbs->attr->num_of_specs;
+ ++verbs->attr.num_of_specs;
verbs->size += size;
}
@@ -392,9 +392,9 @@
if (!(item_flags & l2m))
flow_verbs_spec_add(&dev_flow->verbs, ð, size);
else
- flow_verbs_item_vlan_update(dev_flow->verbs.attr, ð);
+ flow_verbs_item_vlan_update(&dev_flow->verbs.attr, ð);
if (!tunnel)
- dev_flow->handle.vf_vlan.tag =
+ dev_flow->handle->vf_vlan.tag =
rte_be_to_cpu_16(spec->tci) & 0x0fff;
}
@@ -744,7 +744,7 @@
const struct rte_flow_item *item __rte_unused,
uint64_t item_flags)
{
- struct mlx5_flow_resource_verbs *verbs = &dev_flow->verbs;
+ struct mlx5_flow_verbs_workspace *verbs = &dev_flow->verbs;
#ifndef HAVE_IBV_DEVICE_MPLS_SUPPORT
unsigned int size = sizeof(struct ibv_flow_spec_tunnel);
struct ibv_flow_spec_tunnel tunnel = {
@@ -774,11 +774,11 @@
}
#endif
if (item_flags & MLX5_FLOW_LAYER_OUTER_L3_IPV4)
- flow_verbs_item_gre_ip_protocol_update(verbs->attr,
+ flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
IBV_FLOW_SPEC_IPV4_EXT,
IPPROTO_GRE);
else
- flow_verbs_item_gre_ip_protocol_update(verbs->attr,
+ flow_verbs_item_gre_ip_protocol_update(&verbs->attr,
IBV_FLOW_SPEC_IPV6,
IPPROTO_GRE);
flow_verbs_spec_add(verbs, &tunnel, size);
@@ -1385,6 +1385,8 @@
* The required size is calculate based on the actions and items. This function
* also returns the detected actions and items for later use.
*
+ * @param[in] dev
+ * Pointer to Ethernet device.
* @param[in] attr
* Pointer to the flow attributes.
* @param[in] items
@@ -1399,25 +1401,45 @@
* is set.
*/
static struct mlx5_flow *
-flow_verbs_prepare(const struct rte_flow_attr *attr __rte_unused,
+flow_verbs_prepare(struct rte_eth_dev *dev,
+ const struct rte_flow_attr *attr __rte_unused,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
struct rte_flow_error *error)
{
- size_t size = sizeof(struct mlx5_flow) + sizeof(struct ibv_flow_attr);
+ size_t size = 0;
struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *dev_handle;
+ struct mlx5_priv *priv = dev->data->dev_private;
size += flow_verbs_get_actions_size(actions);
size += flow_verbs_get_items_size(items);
- dev_flow = rte_calloc(__func__, 1, size, 0);
- if (!dev_flow) {
+ if (size > MLX5_VERBS_MAX_SPEC_ACT_SIZE) {
+ rte_flow_error_set(error, E2BIG,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "Verbs spec/action size too large");
+ return NULL;
+ }
+ /* In case of corrupting the memory. */
+ if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
+ rte_flow_error_set(error, ENOSPC,
+ RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
+ "not free temporary device flow");
+ return NULL;
+ }
+ dev_handle = rte_calloc(__func__, 1, MLX5_FLOW_HANDLE_VERBS_SIZE, 0);
+ if (!dev_handle) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
- "not enough memory to create flow");
+ "not enough memory to create flow handle");
return NULL;
}
- dev_flow->verbs.attr = (void *)(dev_flow + 1);
- dev_flow->verbs.specs = (void *)(dev_flow->verbs.attr + 1);
+ /* No multi-thread supporting. */
+ dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
+ dev_flow->handle = dev_handle;
+ /* Memcpy is used, only size needs to be cleared to 0. */
+ dev_flow->verbs.size = 0;
+ dev_flow->verbs.attr.num_of_specs = 0;
dev_flow->ingress = attr->ingress;
/* Need to set transfer attribute: not supported in Verbs mode. */
return dev_flow;
@@ -1499,7 +1521,7 @@
"action not supported");
}
}
- dev_flow->handle.act_flags = action_flags;
+ dev_flow->handle->act_flags = action_flags;
for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
@@ -1601,10 +1623,11 @@
"item not supported");
}
}
- dev_flow->handle.layers = item_flags;
- dev_flow->verbs.attr->priority =
+ dev_flow->handle->layers = item_flags;
+ /* Other members of attr will be ignored. */
+ dev_flow->verbs.attr.priority =
mlx5_flow_adjust_priority(dev, priority, subpriority);
- dev_flow->verbs.attr->port = (uint8_t)priv->ibv_port;
+ dev_flow->verbs.attr.port = (uint8_t)priv->ibv_port;
return 0;
}
@@ -1619,26 +1642,24 @@
static void
flow_verbs_remove(struct rte_eth_dev *dev, struct rte_flow *flow)
{
- struct mlx5_flow_handle *dh;
- struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *handle;
if (!flow)
return;
- LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
- dh = &dev_flow->handle;
- if (dh->ib_flow) {
- claim_zero(mlx5_glue->destroy_flow(dh->ib_flow));
- dh->ib_flow = NULL;
+ LIST_FOREACH(handle, &flow->dev_handles, next) {
+ if (handle->ib_flow) {
+ claim_zero(mlx5_glue->destroy_flow(handle->ib_flow));
+ handle->ib_flow = NULL;
}
- if (dh->hrxq) {
- if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP)
+ if (handle->hrxq) {
+ if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
mlx5_hrxq_drop_release(dev);
else
- mlx5_hrxq_release(dev, dh->hrxq);
- dh->hrxq = NULL;
+ mlx5_hrxq_release(dev, handle->hrxq);
+ handle->hrxq = NULL;
}
- if (dh->vf_vlan.tag && dh->vf_vlan.created)
- mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
+ if (handle->vf_vlan.tag && handle->vf_vlan.created)
+ mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
}
}
@@ -1653,15 +1674,15 @@
static void
flow_verbs_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
{
- struct mlx5_flow *dev_flow;
+ struct mlx5_flow_handle *handle;
if (!flow)
return;
flow_verbs_remove(dev, flow);
- while (!LIST_EMPTY(&flow->dev_flows)) {
- dev_flow = LIST_FIRST(&flow->dev_flows);
- LIST_REMOVE(dev_flow, next);
- rte_free(dev_flow);
+ while (!LIST_EMPTY(&flow->dev_handles)) {
+ handle = LIST_FIRST(&flow->dev_handles);
+ LIST_REMOVE(handle, next);
+ rte_free(handle);
}
if (flow->counter) {
flow_verbs_counter_release(dev, flow->counter);
@@ -1687,15 +1708,17 @@
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_flow_handle *dh;
+ struct mlx5_flow_handle *handle;
struct mlx5_flow *dev_flow;
int err;
-
- LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
- dh = &dev_flow->handle;
- if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP) {
- dh->hrxq = mlx5_hrxq_drop_new(dev);
- if (!dh->hrxq) {
+ int idx;
+
+ for (idx = priv->flow_idx - 1; idx >= 0; idx--) {
+ dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
+ handle = dev_flow->handle;
+ if (handle->act_flags & MLX5_FLOW_ACTION_DROP) {
+ handle->hrxq = mlx5_hrxq_drop_new(dev);
+ if (!handle->hrxq) {
rte_flow_error_set
(error, errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -1717,7 +1740,7 @@
dev_flow->hash_fields,
(*flow->rss.queue),
flow->rss.queue_num,
- !!(dev_flow->handle.layers &
+ !!(handle->layers &
MLX5_FLOW_LAYER_TUNNEL));
if (!hrxq) {
rte_flow_error_set
@@ -1726,11 +1749,11 @@
"cannot get hash queue");
goto error;
}
- dh->hrxq = hrxq;
+ handle->hrxq = hrxq;
}
- dh->ib_flow = mlx5_glue->create_flow(dh->hrxq->qp,
- dev_flow->verbs.attr);
- if (!dh->ib_flow) {
+ handle->ib_flow = mlx5_glue->create_flow(handle->hrxq->qp,
+ &dev_flow->verbs.attr);
+ if (!handle->ib_flow) {
rte_flow_error_set(error, errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
@@ -1738,31 +1761,29 @@
goto error;
}
if (priv->vmwa_context &&
- dev_flow->handle.vf_vlan.tag &&
- !dev_flow->handle.vf_vlan.created) {
+ handle->vf_vlan.tag && !handle->vf_vlan.created) {
/*
* The rule contains the VLAN pattern.
* For VF we are going to create VLAN
* interface to make hypervisor set correct
* e-Switch vport context.
*/
- mlx5_vlan_vmwa_acquire(dev, &dev_flow->handle.vf_vlan);
+ mlx5_vlan_vmwa_acquire(dev, &handle->vf_vlan);
}
}
return 0;
error:
err = rte_errno; /* Save rte_errno before cleanup. */
- LIST_FOREACH(dev_flow, &flow->dev_flows, next) {
- dh = &dev_flow->handle;
- if (dh->hrxq) {
- if (dev_flow->handle.act_flags & MLX5_FLOW_ACTION_DROP)
+ LIST_FOREACH(handle, &flow->dev_handles, next) {
+ if (handle->hrxq) {
+ if (handle->act_flags & MLX5_FLOW_ACTION_DROP)
mlx5_hrxq_drop_release(dev);
else
- mlx5_hrxq_release(dev, dh->hrxq);
- dh->hrxq = NULL;
+ mlx5_hrxq_release(dev, handle->hrxq);
+ handle->hrxq = NULL;
}
- if (dh->vf_vlan.tag && dh->vf_vlan.created)
- mlx5_vlan_vmwa_release(dev, &dh->vf_vlan);
+ if (handle->vf_vlan.tag && handle->vf_vlan.created)
+ mlx5_vlan_vmwa_release(dev, &handle->vf_vlan);
}
rte_errno = err; /* Restore rte_errno. */
return -rte_errno;
@@ -307,6 +307,7 @@
mlx5_txq_stop(dev);
return -rte_errno;
}
+ /* Set started flag here for the following steps like control flow. */
dev->data->dev_started = 1;
ret = mlx5_rx_intr_vec_enable(dev);
if (ret) {