[v2,13/25] net/mlx5: make flow table cache thread safe
diff mbox series

Message ID 1603437295-119083-14-git-send-email-suanmingm@nvidia.com
State Superseded
Delegated to: Raslan Darawsheh
Headers show
Series
  • *net/mlx5: support multiple-thread flow operations
Related show

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Suanming Mou Oct. 23, 2020, 7:14 a.m. UTC
From: Xueming Li <xuemingl@nvidia.com>

To support multi-thread flow insertion/removal, this patch uses thread
safe hash list API for flow table cache hash list.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 drivers/net/mlx5/mlx5.c         | 102 ++++---------------------
 drivers/net/mlx5/mlx5.h         |   2 +-
 drivers/net/mlx5/mlx5_flow.h    |  17 +++++
 drivers/net/mlx5/mlx5_flow_dv.c | 164 ++++++++++++++++++++--------------------
 4 files changed, 116 insertions(+), 169 deletions(-)

Patch
diff mbox series

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index da043e2..fa769cd 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1003,7 +1003,7 @@  struct mlx5_dev_ctx_shared *
 }
 
 /**
- * Destroy table hash list and all the root entries per domain.
+ * Destroy table hash list.
  *
  * @param[in] priv
  *   Pointer to the private device data structure.
@@ -1012,46 +1012,9 @@  struct mlx5_dev_ctx_shared *
 mlx5_free_table_hash_list(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_flow_tbl_data_entry *tbl_data;
-	union mlx5_flow_tbl_key table_key = {
-		{
-			.table_id = 0,
-			.reserved = 0,
-			.domain = 0,
-			.direction = 0,
-		}
-	};
-	struct mlx5_hlist_entry *pos;
 
 	if (!sh->flow_tbls)
 		return;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
-	table_key.direction = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
-	table_key.direction = 0;
-	table_key.domain = 1;
-	pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64, NULL);
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		MLX5_ASSERT(tbl_data);
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_free(tbl_data);
-	}
 	mlx5_hlist_destroy(sh->flow_tbls);
 }
 
@@ -1066,80 +1029,45 @@  struct mlx5_dev_ctx_shared *
  *   Zero on success, positive error code otherwise.
  */
 int
-mlx5_alloc_table_hash_list(struct mlx5_priv *priv)
+mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 {
+	int err = 0;
+	/* Tables are only used in DV and DR modes. */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
 	char s[MLX5_HLIST_NAMESIZE];
-	int err = 0;
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, NULL, NULL, NULL);
+					  0, 0, flow_dv_tbl_create_cb, NULL,
+					  flow_dv_tbl_remove_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
+	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
+	struct rte_flow_error error;
+	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
+
 	/*
 	 * In case we have not DR support, the zero tables should be created
 	 * because DV expect to see them even if they cannot be created by
 	 * RDMA-CORE.
 	 */
-	union mlx5_flow_tbl_key table_key = {
-		{
-			.table_id = 0,
-			.reserved = 0,
-			.domain = 0,
-			.direction = 0,
-		}
-	};
-	struct mlx5_flow_tbl_data_entry *tbl_data = mlx5_malloc(MLX5_MEM_ZERO,
-							  sizeof(*tbl_data), 0,
-							  SOCKET_ID_ANY);
-
-	if (!tbl_data) {
-		err = ENOMEM;
-		goto error;
-	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	rte_atomic32_init(&tbl_data->tbl.refcnt);
-	rte_atomic32_inc(&tbl_data->tbl.refcnt);
-	table_key.direction = 1;
-	tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
-			       SOCKET_ID_ANY);
-	if (!tbl_data) {
+	if (!flow_dv_tbl_resource_get(dev, 0, 0, 0, 1, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 1, 0, 1, &error) ||
+	    !flow_dv_tbl_resource_get(dev, 0, 0, 1, 1, &error)) {
 		err = ENOMEM;
 		goto error;
 	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	rte_atomic32_init(&tbl_data->tbl.refcnt);
-	rte_atomic32_inc(&tbl_data->tbl.refcnt);
-	table_key.direction = 0;
-	table_key.domain = 1;
-	tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
-			       SOCKET_ID_ANY);
-	if (!tbl_data) {
-		err = ENOMEM;
-		goto error;
-	}
-	tbl_data->entry.key = table_key.v64;
-	err = mlx5_hlist_insert(sh->flow_tbls, &tbl_data->entry);
-	if (err)
-		goto error;
-	rte_atomic32_init(&tbl_data->tbl.refcnt);
-	rte_atomic32_inc(&tbl_data->tbl.refcnt);
 	return err;
 error:
 	mlx5_free_table_hash_list(priv);
 #endif /* HAVE_MLX5DV_DR */
+#endif
 	return err;
 }
 
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index ed16b0d..ae6d37d 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -496,7 +496,7 @@  struct mlx5_dev_shared_port {
 	struct {
 		/* Table ID should be at the lowest address. */
 		uint32_t table_id;	/**< ID of the table. */
-		uint16_t reserved;	/**< must be zero for comparison. */
+		uint16_t dummy;		/**< Dummy table for DV API. */
 		uint8_t domain;		/**< 1 - FDB, 0 - NIC TX/RX. */
 		uint8_t direction;	/**< 1 - egress, 0 - ingress. */
 	};
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 3d325b2..8d12b5d 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -381,6 +381,13 @@  enum mlx5_flow_fate_type {
 	MLX5_FLOW_FATE_MAX,
 };
 
+/* Hash list callback context */
+struct mlx5_flow_cb_ctx {
+	struct rte_eth_dev *dev;
+	struct rte_flow_error *error;
+	void *data;
+};
+
 /* Matcher PRM representation */
 struct mlx5_flow_dv_match_params {
 	size_t size;
@@ -1149,4 +1156,14 @@  int mlx5_flow_destroy_policer_rules(struct rte_eth_dev *dev,
 int mlx5_flow_meter_flush(struct rte_eth_dev *dev,
 			  struct rte_mtr_error *error);
 int mlx5_flow_dv_discover_counter_offset_support(struct rte_eth_dev *dev);
+
+/* Hash list callbacks for flow tables: */
+struct mlx5_hlist_entry *flow_dv_tbl_create_cb(struct mlx5_hlist *list,
+					       uint64_t key, void *entry_ctx);
+void flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
+			   struct mlx5_hlist_entry *entry);
+struct mlx5_flow_tbl_resource *flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
+		uint32_t table_id, uint8_t egress, uint8_t transfer,
+		uint8_t dummy, struct rte_flow_error *error);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 43d16b4..18a52b0 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -7800,54 +7800,18 @@  struct field_modify_info modify_tcp[] = {
 }
 
 
-/**
- * Get a flow table.
- *
- * @param[in, out] dev
- *   Pointer to rte_eth_dev structure.
- * @param[in] table_id
- *   Table id to use.
- * @param[in] egress
- *   Direction of the table.
- * @param[in] transfer
- *   E-Switch or NIC flow.
- * @param[out] error
- *   pointer to error structure.
- *
- * @return
- *   Returns tables resource based on the index, NULL in case of failed.
- */
-static struct mlx5_flow_tbl_resource *
-flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
-			 uint32_t table_id, uint8_t egress,
-			 uint8_t transfer,
-			 struct rte_flow_error *error)
+struct mlx5_hlist_entry *
+flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *ctx)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	struct mlx5_flow_tbl_resource *tbl;
-	union mlx5_flow_tbl_key table_key = {
-		{
-			.table_id = table_id,
-			.reserved = 0,
-			.domain = !!transfer,
-			.direction = !!egress,
-		}
-	};
-	struct mlx5_hlist_entry *pos = mlx5_hlist_lookup(sh->flow_tbls,
-							 table_key.v64, NULL);
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
 	struct mlx5_flow_tbl_data_entry *tbl_data;
+	struct rte_flow_error *error = ctx;
+	union mlx5_flow_tbl_key key = { .v64 = key64 };
+	struct mlx5_flow_tbl_resource *tbl;
+	void *domain;
 	uint32_t idx = 0;
 	int ret;
-	void *domain;
 
-	if (pos) {
-		tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
-					entry);
-		tbl = &tbl_data->tbl;
-		rte_atomic32_inc(&tbl->refcnt);
-		return tbl;
-	}
 	tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
 	if (!tbl_data) {
 		rte_flow_error_set(error, ENOMEM,
@@ -7858,14 +7822,15 @@  struct field_modify_info modify_tcp[] = {
 	}
 	tbl_data->idx = idx;
 	tbl = &tbl_data->tbl;
-	pos = &tbl_data->entry;
-	if (transfer)
+	if (key.dummy)
+		return &tbl_data->entry;
+	if (key.domain)
 		domain = sh->fdb_domain;
-	else if (egress)
+	else if (key.direction)
 		domain = sh->tx_domain;
 	else
 		domain = sh->rx_domain;
-	ret = mlx5_flow_os_create_flow_tbl(domain, table_id, &tbl->obj);
+	ret = mlx5_flow_os_create_flow_tbl(domain, key.table_id, &tbl->obj);
 	if (ret) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -7873,12 +7838,7 @@  struct field_modify_info modify_tcp[] = {
 		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
 		return NULL;
 	}
-	/*
-	 * No multi-threads now, but still better to initialize the reference
-	 * count before insert it into the hash list.
-	 */
-	rte_atomic32_init(&tbl->refcnt);
-	if (table_id) {
+	if (key.table_id) {
 		ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
 					(tbl->obj, &tbl_data->jump.action);
 		if (ret) {
@@ -7891,17 +7851,69 @@  struct field_modify_info modify_tcp[] = {
 			return NULL;
 		}
 	}
-	pos->key = table_key.v64;
-	ret = !mlx5_hlist_insert(sh->flow_tbls, pos);
-	if (ret < 0) {
-		rte_flow_error_set(error, -ret,
+	return &tbl_data->entry;
+}
+
+/**
+ * Get a flow table.
+ *
+ * @param[in, out] dev
+ *   Pointer to rte_eth_dev structure.
+ * @param[in] table_id
+ *   Table id to use.
+ * @param[in] egress
+ *   Direction of the table.
+ * @param[in] transfer
+ *   E-Switch or NIC flow.
+ * @param[in] dummy
+ *   Dummy entry for dv API.
+ * @param[out] error
+ *   pointer to error structure.
+ *
+ * @return
+ *   Returns tables resource based on the index, NULL in case of failed.
+ */
+struct mlx5_flow_tbl_resource *
+flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
+			 uint32_t table_id, uint8_t egress,
+			 uint8_t transfer, uint8_t dummy,
+			 struct rte_flow_error *error)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	union mlx5_flow_tbl_key table_key = {
+		{
+			.table_id = table_id,
+			.dummy = dummy,
+			.domain = !!transfer,
+			.direction = !!egress,
+		}
+	};
+	struct mlx5_hlist_entry *entry;
+	struct mlx5_flow_tbl_data_entry *tbl_data;
+
+	entry = mlx5_hlist_register(priv->sh->flow_tbls, table_key.v64, NULL);
+	if (!entry) {
+		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
-				   "cannot insert flow table data entry");
-		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
-		mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
+				   "cannot get table");
+		return NULL;
 	}
-	rte_atomic32_inc(&tbl->refcnt);
-	return tbl;
+	tbl_data = container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+	return &tbl_data->tbl;
+}
+
+void
+flow_dv_tbl_remove_cb(struct mlx5_hlist *list,
+		      struct mlx5_hlist_entry *entry)
+{
+	struct mlx5_dev_ctx_shared *sh = list->ctx;
+	struct mlx5_flow_tbl_data_entry *tbl_data =
+		container_of(entry, struct mlx5_flow_tbl_data_entry, entry);
+
+	MLX5_ASSERT(entry && sh);
+	if (tbl_data->tbl.obj)
+		mlx5_flow_os_destroy_flow_tbl(tbl_data->tbl.obj);
+	mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], tbl_data->idx);
 }
 
 /**
@@ -7926,18 +7938,7 @@  struct field_modify_info modify_tcp[] = {
 
 	if (!tbl)
 		return 0;
-	if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
-		struct mlx5_hlist_entry *pos = &tbl_data->entry;
-
-		mlx5_flow_os_destroy_flow_tbl(tbl->obj);
-		tbl->obj = NULL;
-		/* remove the entry from the hash list and free memory. */
-		mlx5_hlist_remove(sh->flow_tbls, pos);
-		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_JUMP],
-				tbl_data->idx);
-		return 0;
-	}
-	return 1;
+	return mlx5_hlist_unregister(sh->flow_tbls, &tbl_data->entry);
 }
 
 /**
@@ -7976,7 +7977,7 @@  struct field_modify_info modify_tcp[] = {
 	int ret;
 
 	tbl = flow_dv_tbl_resource_get(dev, key->table_id, key->direction,
-				       key->domain, error);
+				       key->domain, 0, error);
 	if (!tbl)
 		return -rte_errno;	/* No need to refill the error info */
 	tbl_data = container_of(tbl, struct mlx5_flow_tbl_data_entry, tbl);
@@ -8471,7 +8472,7 @@  struct field_modify_info modify_tcp[] = {
 	*cache_resource = *resource;
 	/* Create normal path table level */
 	tbl = flow_dv_tbl_resource_get(dev, next_ft_id,
-					attr->egress, attr->transfer, error);
+				       attr->egress, attr->transfer, 0, error);
 	if (!tbl) {
 		rte_flow_error_set(error, ENOMEM,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -9377,7 +9378,8 @@  struct field_modify_info modify_tcp[] = {
 				return ret;
 			tbl = flow_dv_tbl_resource_get(dev, table,
 						       attr->egress,
-						       attr->transfer, error);
+						       attr->transfer, 0,
+						       error);
 			if (!tbl)
 				return rte_flow_error_set
 						(error, errno,
@@ -10771,7 +10773,7 @@  struct field_modify_info modify_tcp[] = {
 		dtb = &mtb->ingress;
 	/* Create the meter table with METER level. */
 	dtb->tbl = flow_dv_tbl_resource_get(dev, MLX5_FLOW_TABLE_LEVEL_METER,
-					    egress, transfer, &error);
+					    egress, transfer, 0, &error);
 	if (!dtb->tbl) {
 		DRV_LOG(ERR, "Failed to create meter policer table.");
 		return -1;
@@ -10779,7 +10781,7 @@  struct field_modify_info modify_tcp[] = {
 	/* Create the meter suffix table with SUFFIX level. */
 	dtb->sfx_tbl = flow_dv_tbl_resource_get(dev,
 					    MLX5_FLOW_TABLE_LEVEL_SUFFIX,
-					    egress, transfer, &error);
+					    egress, transfer, 0, &error);
 	if (!dtb->sfx_tbl) {
 		DRV_LOG(ERR, "Failed to create meter suffix table.");
 		return -1;
@@ -11098,10 +11100,10 @@  struct field_modify_info modify_tcp[] = {
 	void *flow = NULL;
 	int i, ret = -1;
 
-	tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, NULL);
+	tbl = flow_dv_tbl_resource_get(dev, 0, 0, 0, 0, NULL);
 	if (!tbl)
 		goto err;
-	dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, NULL);
+	dest_tbl = flow_dv_tbl_resource_get(dev, 1, 0, 0, 0, NULL);
 	if (!dest_tbl)
 		goto err;
 	dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0x4);