diff mbox series

[v3,06/17] net/mlx5: add extended metadata mode for hardware steering

Message ID 20220930125315.5079-7-suanmingm@nvidia.com (mailing list archive)
State Superseded, archived
Delegated to: Raslan Darawsheh
Headers show
Series net/mlx5: HW steering PMD update | expand

Checks

Context Check Description
ci/checkpatch warning coding style issues

Commit Message

Suanming Mou Sept. 30, 2022, 12:53 p.m. UTC
From: Bing Zhao <bingz@nvidia.com>

The new mode 4 of devarg "dv_xmeta_en" is added for HWS only. In this
mode, the Rx / Tx metadata with 32b width copy between FDB and NIC is
supported. The mark is only supported in NIC and there is no copy
supported.

Signed-off-by: Bing Zhao <bingz@nvidia.com>
---
 drivers/net/mlx5/linux/mlx5_os.c |  10 +-
 drivers/net/mlx5/mlx5.c          |   7 +-
 drivers/net/mlx5/mlx5.h          |   8 +-
 drivers/net/mlx5/mlx5_flow.c     |   8 +-
 drivers/net/mlx5/mlx5_flow.h     |  14 +
 drivers/net/mlx5/mlx5_flow_dv.c  |  43 +-
 drivers/net/mlx5/mlx5_flow_hw.c  | 864 ++++++++++++++++++++++++++++---
 drivers/net/mlx5/mlx5_trigger.c  |   3 +
 8 files changed, 872 insertions(+), 85 deletions(-)
diff mbox series

Patch

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index e0586a4d6f..061b825e7b 100644
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -1554,6 +1554,15 @@  mlx5_dev_spawn(struct rte_device *dpdk_dev,
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 		if (priv->vport_meta_mask)
 			flow_hw_set_port_info(eth_dev);
+		if (priv->sh->config.dv_esw_en &&
+		    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
+		    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) {
+			DRV_LOG(ERR,
+				"metadata mode %u is not supported in HWS eswitch mode",
+				priv->sh->config.dv_xmeta_en);
+				err = ENOTSUP;
+				goto error;
+		}
 		/* Only HWS requires this information. */
 		flow_hw_init_tags_set(eth_dev);
 		if (priv->sh->config.dv_esw_en &&
@@ -1569,7 +1578,6 @@  mlx5_dev_spawn(struct rte_device *dpdk_dev,
 		goto error;
 #endif
 	}
-	/* Port representor shares the same max priority with pf port. */
 	if (!priv->sh->flow_priority_check_flag) {
 		/* Supported Verbs flow priority number detection. */
 		err = mlx5_flow_discover_priorities(eth_dev);
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 74adb677f4..cf5146d677 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -1218,7 +1218,8 @@  mlx5_dev_args_check_handler(const char *key, const char *val, void *opaque)
 		if (tmp != MLX5_XMETA_MODE_LEGACY &&
 		    tmp != MLX5_XMETA_MODE_META16 &&
 		    tmp != MLX5_XMETA_MODE_META32 &&
-		    tmp != MLX5_XMETA_MODE_MISS_INFO) {
+		    tmp != MLX5_XMETA_MODE_MISS_INFO &&
+		    tmp != MLX5_XMETA_MODE_META32_HWS) {
 			DRV_LOG(ERR, "Invalid extensive metadata parameter.");
 			rte_errno = EINVAL;
 			return -rte_errno;
@@ -2849,6 +2850,10 @@  mlx5_set_metadata_mask(struct rte_eth_dev *dev)
 		meta = UINT32_MAX;
 		mark = (reg_c0 >> rte_bsf32(reg_c0)) & MLX5_FLOW_MARK_MASK;
 		break;
+	case MLX5_XMETA_MODE_META32_HWS:
+		meta = UINT32_MAX;
+		mark = MLX5_FLOW_MARK_MASK;
+		break;
 	default:
 		meta = 0;
 		mark = 0;
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 0bf21c1efe..fc4bc4e6a3 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -298,8 +298,8 @@  struct mlx5_sh_config {
 	uint32_t reclaim_mode:2; /* Memory reclaim mode. */
 	uint32_t dv_esw_en:1; /* Enable E-Switch DV flow. */
 	/* Enable DV flow. 1 means SW steering, 2 means HW steering. */
-	unsigned int dv_flow_en:2;
-	uint32_t dv_xmeta_en:2; /* Enable extensive flow metadata. */
+	uint32_t dv_flow_en:2; /* Enable DV flow. */
+	uint32_t dv_xmeta_en:3; /* Enable extensive flow metadata. */
 	uint32_t dv_miss_info:1; /* Restore packet after partial hw miss. */
 	uint32_t l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */
 	uint32_t vf_nl_en:1; /* Enable Netlink requests in VF mode. */
@@ -312,7 +312,6 @@  struct mlx5_sh_config {
 	uint32_t fdb_def_rule:1; /* Create FDB default jump rule */
 };
 
-
 /* Structure for VF VLAN workaround. */
 struct mlx5_vf_vlan {
 	uint32_t tag:12;
@@ -1279,12 +1278,12 @@  struct mlx5_dev_ctx_shared {
 	struct mlx5_lb_ctx self_lb; /* QP to enable self loopback for Devx. */
 	unsigned int flow_max_priority;
 	enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+	/* Availability of mreg_c's. */
 	void *devx_channel_lwm;
 	struct rte_intr_handle *intr_handle_lwm;
 	pthread_mutex_t lwm_config_lock;
 	uint32_t host_shaper_rate:8;
 	uint32_t lwm_triggered:1;
-	/* Availability of mreg_c's. */
 	struct mlx5_dev_shared_port port[]; /* per device port data array. */
 };
 
@@ -1508,6 +1507,7 @@  struct mlx5_priv {
 	struct rte_flow_template_table *hw_esw_sq_miss_root_tbl;
 	struct rte_flow_template_table *hw_esw_sq_miss_tbl;
 	struct rte_flow_template_table *hw_esw_zero_tbl;
+	struct rte_flow_template_table *hw_tx_meta_cpy_tbl;
 	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
 	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 9c44b2e99b..b570ed7f69 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -1107,6 +1107,8 @@  mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 			return REG_C_0;
 		case MLX5_XMETA_MODE_META32:
 			return REG_C_1;
+		case MLX5_XMETA_MODE_META32_HWS:
+			return REG_C_1;
 		}
 		break;
 	case MLX5_METADATA_TX:
@@ -1119,11 +1121,14 @@  mlx5_flow_get_reg_id(struct rte_eth_dev *dev,
 			return REG_C_0;
 		case MLX5_XMETA_MODE_META32:
 			return REG_C_1;
+		case MLX5_XMETA_MODE_META32_HWS:
+			return REG_C_1;
 		}
 		break;
 	case MLX5_FLOW_MARK:
 		switch (config->dv_xmeta_en) {
 		case MLX5_XMETA_MODE_LEGACY:
+		case MLX5_XMETA_MODE_META32_HWS:
 			return REG_NON;
 		case MLX5_XMETA_MODE_META16:
 			return REG_C_1;
@@ -4442,7 +4447,8 @@  static bool flow_check_modify_action_type(struct rte_eth_dev *dev,
 		return true;
 	case RTE_FLOW_ACTION_TYPE_FLAG:
 	case RTE_FLOW_ACTION_TYPE_MARK:
-		if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY)
+		if (priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_LEGACY &&
+		    priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS)
 			return true;
 		else
 			return false;
diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 93f0e189d4..a8b27ea494 100644
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -48,6 +48,12 @@  enum mlx5_rte_flow_action_type {
 	MLX5_RTE_FLOW_ACTION_TYPE_RSS,
 };
 
+/* Private (internal) Field IDs for MODIFY_FIELD action. */
+enum mlx5_rte_flow_field_id {
+		MLX5_RTE_FLOW_FIELD_END = INT_MIN,
+			MLX5_RTE_FLOW_FIELD_META_REG,
+};
+
 #define MLX5_INDIRECT_ACTION_TYPE_OFFSET 30
 
 enum {
@@ -1167,6 +1173,7 @@  struct rte_flow_actions_template {
 	struct rte_flow_action *masks; /* Cached action masks.*/
 	uint16_t mhdr_off; /* Offset of DR modify header action. */
 	uint32_t refcnt; /* Reference counter. */
+	uint16_t rx_cpy_pos; /* Action position of Rx metadata to be copied. */
 };
 
 /* Jump action struct. */
@@ -1243,6 +1250,11 @@  struct mlx5_flow_group {
 #define MLX5_HW_TBL_MAX_ITEM_TEMPLATE 2
 #define MLX5_HW_TBL_MAX_ACTION_TEMPLATE 32
 
+struct mlx5_flow_template_table_cfg {
+	struct rte_flow_template_table_attr attr; /* Table attributes passed through flow API. */
+	bool external; /* True if created by flow API, false if table is internal to PMD. */
+};
+
 struct rte_flow_template_table {
 	LIST_ENTRY(rte_flow_template_table) next;
 	struct mlx5_flow_group *grp; /* The group rte_flow_template_table uses. */
@@ -1252,6 +1264,7 @@  struct rte_flow_template_table {
 	/* Action templates bind to the table. */
 	struct mlx5_hw_action_template ats[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
 	struct mlx5_indexed_pool *flow; /* The table's flow ipool. */
+	struct mlx5_flow_template_table_cfg cfg;
 	uint32_t type; /* Flow table type RX/TX/FDB. */
 	uint8_t nb_item_templates; /* Item template number. */
 	uint8_t nb_action_templates; /* Action template number. */
@@ -2333,4 +2346,5 @@  int mlx5_flow_hw_esw_create_mgr_sq_miss_flow(struct rte_eth_dev *dev);
 int mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev,
 					 uint32_t txq);
 int mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev);
+int mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev);
 #endif /* RTE_PMD_MLX5_FLOW_H_ */
diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 3fc2453045..5b72cfaa61 100644
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -1783,7 +1783,8 @@  mlx5_flow_field_id_to_modify_info
 			int reg;
 
 			if (priv->sh->config.dv_flow_en == 2)
-				reg = REG_C_1;
+				reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG,
+							 data->level);
 			else
 				reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG,
 							   data->level, error);
@@ -1862,6 +1863,24 @@  mlx5_flow_field_id_to_modify_info
 		else
 			info[idx].offset = off_be;
 		break;
+	case MLX5_RTE_FLOW_FIELD_META_REG:
+		{
+			uint32_t meta_mask = priv->sh->dv_meta_mask;
+			uint32_t meta_count = __builtin_popcount(meta_mask);
+			uint32_t reg = data->level;
+
+			RTE_SET_USED(meta_count);
+			MLX5_ASSERT(data->offset + width <= meta_count);
+			MLX5_ASSERT(reg != REG_NON);
+			MLX5_ASSERT(reg < RTE_DIM(reg_to_field));
+			info[idx] = (struct field_modify_info){4, 0, reg_to_field[reg]};
+			if (mask)
+				mask[idx] = flow_modify_info_mask_32_masked
+					(width, data->offset, meta_mask);
+			else
+				info[idx].offset = data->offset;
+		}
+		break;
 	case RTE_FLOW_FIELD_POINTER:
 	case RTE_FLOW_FIELD_VALUE:
 	default:
@@ -9819,7 +9838,19 @@  flow_dv_translate_item_meta(struct rte_eth_dev *dev,
 	mask = meta_m->data;
 	if (key_type == MLX5_SET_MATCHER_HS_M)
 		mask = value;
-	reg = flow_dv_get_metadata_reg(dev, attr, NULL);
+	/*
+	 * In the current implementation, REG_B cannot be used to match.
+	 * Force to use REG_C_1 in HWS root table as other tables.
+	 * This map may change.
+	 * NIC: modify - REG_B to be present in SW
+	 *      match - REG_C_1 when copied from FDB, different from SWS
+	 * FDB: modify - REG_C_1 in Xmeta mode, REG_NON in legacy mode
+	 *      match - REG_C_1 in FDB
+	 */
+	if (!!(key_type & MLX5_SET_MATCHER_SW))
+		reg = flow_dv_get_metadata_reg(dev, attr, NULL);
+	else
+		reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_META, 0);
 	if (reg < 0)
 		return;
 	MLX5_ASSERT(reg != REG_NON);
@@ -9919,7 +9950,10 @@  flow_dv_translate_item_tag(struct rte_eth_dev *dev, void *key,
 	/* When set mask, the index should be from spec. */
 	index = tag_vv ? tag_vv->index : tag_v->index;
 	/* Get the metadata register index for the tag. */
-	reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, index, NULL);
+	if (!!(key_type & MLX5_SET_MATCHER_SW))
+		reg = mlx5_flow_get_reg_id(dev, MLX5_APP_TAG, index, NULL);
+	else
+		reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG, index);
 	MLX5_ASSERT(reg > 0);
 	flow_dv_match_meta_reg(key, reg, tag_v->data, tag_m->data);
 }
@@ -13437,7 +13471,8 @@  flow_dv_translate_items_sws(struct rte_eth_dev *dev,
 	 */
 	if (!(wks.item_flags & MLX5_FLOW_ITEM_PORT_ID) &&
 	    !(wks.item_flags & MLX5_FLOW_ITEM_REPRESENTED_PORT) && priv->sh->esw_mode &&
-	    !(attr->egress && !attr->transfer)) {
+	    !(attr->egress && !attr->transfer) &&
+	    attr->group != MLX5_FLOW_MREG_CP_TABLE_GROUP) {
 		if (flow_dv_translate_item_port_id_all(dev, match_mask,
 						   match_value, NULL, attr))
 			return -rte_errno;
diff --git a/drivers/net/mlx5/mlx5_flow_hw.c b/drivers/net/mlx5/mlx5_flow_hw.c
index b3b37f36a2..64d06d4fb4 100644
--- a/drivers/net/mlx5/mlx5_flow_hw.c
+++ b/drivers/net/mlx5/mlx5_flow_hw.c
@@ -20,13 +20,27 @@ 
 /* Default queue to flush the flows. */
 #define MLX5_DEFAULT_FLUSH_QUEUE 0
 
-/* Maximum number of rules in control flow tables */
+/* Maximum number of rules in control flow tables. */
 #define MLX5_HW_CTRL_FLOW_NB_RULES (4096)
 
-/* Flow group for SQ miss default flows/ */
-#define MLX5_HW_SQ_MISS_GROUP (UINT32_MAX)
+/* Lowest flow group usable by an application. */
+#define MLX5_HW_LOWEST_USABLE_GROUP (1)
+
+/* Maximum group index usable by user applications for transfer flows. */
+#define MLX5_HW_MAX_TRANSFER_GROUP (UINT32_MAX - 1)
+
+/* Lowest priority for HW root table. */
+#define MLX5_HW_LOWEST_PRIO_ROOT 15
+
+/* Lowest priority for HW non-root table. */
+#define MLX5_HW_LOWEST_PRIO_NON_ROOT (UINT32_MAX)
 
 static int flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev);
+static int flow_hw_translate_group(struct rte_eth_dev *dev,
+				   const struct mlx5_flow_template_table_cfg *cfg,
+				   uint32_t group,
+				   uint32_t *table_group,
+				   struct rte_flow_error *error);
 
 const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops;
 
@@ -213,12 +227,12 @@  flow_hw_rss_item_flags_get(const struct rte_flow_item items[])
  */
 static struct mlx5_hw_jump_action *
 flow_hw_jump_action_register(struct rte_eth_dev *dev,
-			     const struct rte_flow_attr *attr,
+			     const struct mlx5_flow_template_table_cfg *cfg,
 			     uint32_t dest_group,
 			     struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	struct rte_flow_attr jattr = *attr;
+	struct rte_flow_attr jattr = cfg->attr.flow_attr;
 	struct mlx5_flow_group *grp;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
@@ -226,9 +240,13 @@  flow_hw_jump_action_register(struct rte_eth_dev *dev,
 		.data = &jattr,
 	};
 	struct mlx5_list_entry *ge;
+	uint32_t target_group;
 
-	jattr.group = dest_group;
-	ge = mlx5_hlist_register(priv->sh->flow_tbls, dest_group, &ctx);
+	target_group = dest_group;
+	if (flow_hw_translate_group(dev, cfg, dest_group, &target_group, error))
+		return NULL;
+	jattr.group = target_group;
+	ge = mlx5_hlist_register(priv->sh->flow_tbls, target_group, &ctx);
 	if (!ge)
 		return NULL;
 	grp = container_of(ge, struct mlx5_flow_group, entry);
@@ -760,7 +778,8 @@  flow_hw_modify_field_compile(struct rte_eth_dev *dev,
 				(void *)(uintptr_t)conf->src.pvalue :
 				(void *)(uintptr_t)&conf->src.value;
 		if (conf->dst.field == RTE_FLOW_FIELD_META ||
-		    conf->dst.field == RTE_FLOW_FIELD_TAG) {
+		    conf->dst.field == RTE_FLOW_FIELD_TAG ||
+		    conf->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
 			value = *(const unaligned_uint32_t *)item.spec;
 			value = rte_cpu_to_be_32(value);
 			item.spec = &value;
@@ -860,6 +879,9 @@  flow_hw_represented_port_compile(struct rte_eth_dev *dev,
 	if (m && !!m->port_id) {
 		struct mlx5_priv *port_priv;
 
+		if (!v)
+			return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+						  action, "port index was not provided");
 		port_priv = mlx5_port_to_eswitch_info(v->port_id, false);
 		if (port_priv == NULL)
 			return rte_flow_error_set
@@ -903,8 +925,8 @@  flow_hw_represented_port_compile(struct rte_eth_dev *dev,
  *
  * @param[in] dev
  *   Pointer to the rte_eth_dev structure.
- * @param[in] table_attr
- *   Pointer to the table attributes.
+ * @param[in] cfg
+ *   Pointer to the table configuration.
  * @param[in] item_templates
  *   Item template array to be binded to the table.
  * @param[in/out] acts
@@ -919,12 +941,13 @@  flow_hw_represented_port_compile(struct rte_eth_dev *dev,
  */
 static int
 flow_hw_actions_translate(struct rte_eth_dev *dev,
-			  const struct rte_flow_template_table_attr *table_attr,
+			  const struct mlx5_flow_template_table_cfg *cfg,
 			  struct mlx5_hw_actions *acts,
 			  struct rte_flow_actions_template *at,
 			  struct rte_flow_error *error)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
+	const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
 	const struct rte_flow_attr *attr = &table_attr->flow_attr;
 	struct rte_flow_action *actions = at->actions;
 	struct rte_flow_action *action_start = actions;
@@ -991,7 +1014,7 @@  flow_hw_actions_translate(struct rte_eth_dev *dev,
 					((const struct rte_flow_action_jump *)
 					actions->conf)->group;
 				acts->jump = flow_hw_jump_action_register
-						(dev, attr, jump_group, error);
+						(dev, cfg, jump_group, error);
 				if (!acts->jump)
 					goto err;
 				acts->rule_acts[i].action = (!!attr->group) ?
@@ -1101,6 +1124,16 @@  flow_hw_actions_translate(struct rte_eth_dev *dev,
 							   error);
 			if (err)
 				goto err;
+			/*
+			 * Adjust the action source position for the following.
+			 * ... / MODIFY_FIELD: rx_cpy_pos / (QUEUE|RSS) / ...
+			 * The next action will be Q/RSS, there will not be
+			 * another adjustment and the real source position of
+			 * the following actions will be decreased by 1.
+			 * No change of the total actions in the new template.
+			 */
+			if ((actions - action_start) == at->rx_cpy_pos)
+				action_start += 1;
 			break;
 		case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
 			if (flow_hw_represented_port_compile
@@ -1365,7 +1398,8 @@  flow_hw_modify_field_construct(struct mlx5_hw_q_job *job,
 	else
 		rte_memcpy(values, mhdr_action->src.pvalue, sizeof(values));
 	if (mhdr_action->dst.field == RTE_FLOW_FIELD_META ||
-	    mhdr_action->dst.field == RTE_FLOW_FIELD_TAG) {
+	    mhdr_action->dst.field == RTE_FLOW_FIELD_TAG ||
+	    mhdr_action->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
 		value_p = (unaligned_uint32_t *)values;
 		*value_p = rte_cpu_to_be_32(*value_p);
 	} else if (mhdr_action->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI) {
@@ -1513,7 +1547,7 @@  flow_hw_actions_construct(struct rte_eth_dev *dev,
 			jump_group = ((const struct rte_flow_action_jump *)
 						action->conf)->group;
 			jump = flow_hw_jump_action_register
-				(dev, &attr, jump_group, NULL);
+				(dev, &table->cfg, jump_group, NULL);
 			if (!jump)
 				return -1;
 			rule_acts[act_data->action_dst].action =
@@ -1710,7 +1744,13 @@  flow_hw_async_flow_create(struct rte_eth_dev *dev,
 	job->user_data = user_data;
 	rule_attr.user_data = job;
 	hw_acts = &table->ats[action_template_index].acts;
-	/* Construct the flow actions based on the input actions.*/
+	/*
+	 * Construct the flow actions based on the input actions.
+	 * The implicitly appended action is always fixed, like metadata
+	 * copy action from FDB to NIC Rx.
+	 * No need to copy and contrust a new "actions" list based on the
+	 * user's input, in order to save the cost.
+	 */
 	if (flow_hw_actions_construct(dev, job, hw_acts, pattern_template_index,
 				  actions, rule_acts, &acts_num)) {
 		rte_errno = EINVAL;
@@ -1981,6 +2021,8 @@  flow_hw_q_flow_flush(struct rte_eth_dev *dev,
 	/* Flush flow per-table from MLX5_DEFAULT_FLUSH_QUEUE. */
 	hw_q = &priv->hw_q[MLX5_DEFAULT_FLUSH_QUEUE];
 	LIST_FOREACH(tbl, &priv->flow_hw_tbl, next) {
+		if (!tbl->cfg.external)
+			continue;
 		MLX5_IPOOL_FOREACH(tbl->flow, fidx, flow) {
 			if (flow_hw_async_flow_destroy(dev,
 						MLX5_DEFAULT_FLUSH_QUEUE,
@@ -2018,8 +2060,8 @@  flow_hw_q_flow_flush(struct rte_eth_dev *dev,
  *
  * @param[in] dev
  *   Pointer to the rte_eth_dev structure.
- * @param[in] attr
- *   Pointer to the table attributes.
+ * @param[in] table_cfg
+ *   Pointer to the table configuration.
  * @param[in] item_templates
  *   Item template array to be binded to the table.
  * @param[in] nb_item_templates
@@ -2036,7 +2078,7 @@  flow_hw_q_flow_flush(struct rte_eth_dev *dev,
  */
 static struct rte_flow_template_table *
 flow_hw_table_create(struct rte_eth_dev *dev,
-		     const struct rte_flow_template_table_attr *attr,
+		     const struct mlx5_flow_template_table_cfg *table_cfg,
 		     struct rte_flow_pattern_template *item_templates[],
 		     uint8_t nb_item_templates,
 		     struct rte_flow_actions_template *action_templates[],
@@ -2048,6 +2090,7 @@  flow_hw_table_create(struct rte_eth_dev *dev,
 	struct rte_flow_template_table *tbl = NULL;
 	struct mlx5_flow_group *grp;
 	struct mlx5dr_match_template *mt[MLX5_HW_TBL_MAX_ITEM_TEMPLATE];
+	const struct rte_flow_template_table_attr *attr = &table_cfg->attr;
 	struct rte_flow_attr flow_attr = attr->flow_attr;
 	struct mlx5_flow_cb_ctx ctx = {
 		.dev = dev,
@@ -2088,6 +2131,7 @@  flow_hw_table_create(struct rte_eth_dev *dev,
 	tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl), 0, rte_socket_id());
 	if (!tbl)
 		goto error;
+	tbl->cfg = *table_cfg;
 	/* Allocate flow indexed pool. */
 	tbl->flow = mlx5_ipool_create(&cfg);
 	if (!tbl->flow)
@@ -2131,7 +2175,7 @@  flow_hw_table_create(struct rte_eth_dev *dev,
 			goto at_error;
 		}
 		LIST_INIT(&tbl->ats[i].acts.act_list);
-		err = flow_hw_actions_translate(dev, attr,
+		err = flow_hw_actions_translate(dev, &tbl->cfg,
 						&tbl->ats[i].acts,
 						action_templates[i], error);
 		if (err) {
@@ -2174,6 +2218,96 @@  flow_hw_table_create(struct rte_eth_dev *dev,
 	return NULL;
 }
 
+/**
+ * Translates group index specified by the user in @p attr to internal
+ * group index.
+ *
+ * Translation is done by incrementing group index, so group n becomes n + 1.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] cfg
+ *   Pointer to the template table configuration.
+ * @param[in] group
+ *   Currently used group index (table group or jump destination).
+ * @param[out] table_group
+ *   Pointer to output group index.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   0 on success. Otherwise, returns negative error code, rte_errno is set
+ *   and error structure is filled.
+ */
+static int
+flow_hw_translate_group(struct rte_eth_dev *dev,
+			const struct mlx5_flow_template_table_cfg *cfg,
+			uint32_t group,
+			uint32_t *table_group,
+			struct rte_flow_error *error)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	const struct rte_flow_attr *flow_attr = &cfg->attr.flow_attr;
+
+	if (priv->sh->config.dv_esw_en && cfg->external && flow_attr->transfer) {
+		if (group > MLX5_HW_MAX_TRANSFER_GROUP)
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
+						  NULL,
+						  "group index not supported");
+		*table_group = group + 1;
+	} else {
+		*table_group = group;
+	}
+	return 0;
+}
+
+/**
+ * Create flow table.
+ *
+ * This function is a wrapper over @ref flow_hw_table_create(), which translates parameters
+ * provided by user to proper internal values.
+ *
+ * @param[in] dev
+ *   Pointer to Ethernet device.
+ * @param[in] attr
+ *   Pointer to the table attributes.
+ * @param[in] item_templates
+ *   Item template array to be binded to the table.
+ * @param[in] nb_item_templates
+ *   Number of item templates.
+ * @param[in] action_templates
+ *   Action template array to be binded to the table.
+ * @param[in] nb_action_templates
+ *   Number of action templates.
+ * @param[out] error
+ *   Pointer to error structure.
+ *
+ * @return
+ *   Table on success, Otherwise, returns negative error code, rte_errno is set
+ *   and error structure is filled.
+ */
+static struct rte_flow_template_table *
+flow_hw_template_table_create(struct rte_eth_dev *dev,
+			      const struct rte_flow_template_table_attr *attr,
+			      struct rte_flow_pattern_template *item_templates[],
+			      uint8_t nb_item_templates,
+			      struct rte_flow_actions_template *action_templates[],
+			      uint8_t nb_action_templates,
+			      struct rte_flow_error *error)
+{
+	struct mlx5_flow_template_table_cfg cfg = {
+		.attr = *attr,
+		.external = true,
+	};
+	uint32_t group = attr->flow_attr.group;
+
+	if (flow_hw_translate_group(dev, &cfg, group, &cfg.attr.flow_attr.group, error))
+		return NULL;
+	return flow_hw_table_create(dev, &cfg, item_templates, nb_item_templates,
+				    action_templates, nb_action_templates, error);
+}
+
 /**
  * Destroy flow table.
  *
@@ -2309,10 +2443,13 @@  flow_hw_validate_action_represented_port(struct rte_eth_dev *dev,
 					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 					  "cannot use represented_port actions"
 					  " without an E-Switch");
-	if (mask_conf->port_id) {
+	if (mask_conf && mask_conf->port_id) {
 		struct mlx5_priv *port_priv;
 		struct mlx5_priv *dev_priv;
 
+		if (!action_conf)
+			return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
+						  action, "port index was not provided");
 		port_priv = mlx5_port_to_eswitch_info(action_conf->port_id, false);
 		if (!port_priv)
 			return rte_flow_error_set(error, rte_errno,
@@ -2337,20 +2474,77 @@  flow_hw_validate_action_represented_port(struct rte_eth_dev *dev,
 	return 0;
 }
 
+static inline int
+flow_hw_action_meta_copy_insert(const struct rte_flow_action actions[],
+				const struct rte_flow_action masks[],
+				const struct rte_flow_action *ins_actions,
+				const struct rte_flow_action *ins_masks,
+				struct rte_flow_action *new_actions,
+				struct rte_flow_action *new_masks,
+				uint16_t *ins_pos)
+{
+	uint16_t idx, total = 0;
+	bool ins = false;
+	bool act_end = false;
+
+	MLX5_ASSERT(actions && masks);
+	MLX5_ASSERT(new_actions && new_masks);
+	MLX5_ASSERT(ins_actions && ins_masks);
+	for (idx = 0; !act_end; idx++) {
+		if (idx >= MLX5_HW_MAX_ACTS)
+			return -1;
+		if (actions[idx].type == RTE_FLOW_ACTION_TYPE_RSS ||
+		    actions[idx].type == RTE_FLOW_ACTION_TYPE_QUEUE) {
+			ins = true;
+			*ins_pos = idx;
+		}
+		if (actions[idx].type == RTE_FLOW_ACTION_TYPE_END)
+			act_end = true;
+	}
+	if (!ins)
+		return 0;
+	else if (idx == MLX5_HW_MAX_ACTS)
+		return -1; /* No more space. */
+	total = idx;
+	/* Before the position, no change for the actions. */
+	for (idx = 0; idx < *ins_pos; idx++) {
+		new_actions[idx] = actions[idx];
+		new_masks[idx] = masks[idx];
+	}
+	/* Insert the new action and mask to the position. */
+	new_actions[idx] = *ins_actions;
+	new_masks[idx] = *ins_masks;
+	/* Remaining content is right shifted by one position. */
+	for (; idx < total; idx++) {
+		new_actions[idx + 1] = actions[idx];
+		new_masks[idx + 1] = masks[idx];
+	}
+	return 0;
+}
+
 static int
 flow_hw_action_validate(struct rte_eth_dev *dev,
+			const struct rte_flow_actions_template_attr *attr,
 			const struct rte_flow_action actions[],
 			const struct rte_flow_action masks[],
 			struct rte_flow_error *error)
 {
-	int i;
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint16_t i;
 	bool actions_end = false;
 	int ret;
 
+	/* FDB actions are only valid to proxy port. */
+	if (attr->transfer && (!priv->sh->config.dv_esw_en || !priv->master))
+		return rte_flow_error_set(error, EINVAL,
+					  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+					  NULL,
+					  "transfer actions are only valid to proxy port");
 	for (i = 0; !actions_end; ++i) {
 		const struct rte_flow_action *action = &actions[i];
 		const struct rte_flow_action *mask = &masks[i];
 
+		MLX5_ASSERT(i < MLX5_HW_MAX_ACTS);
 		if (action->type != mask->type)
 			return rte_flow_error_set(error, ENOTSUP,
 						  RTE_FLOW_ERROR_TYPE_ACTION,
@@ -2447,21 +2641,77 @@  flow_hw_actions_template_create(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	int len, act_len, mask_len, i;
-	struct rte_flow_actions_template *at;
+	struct rte_flow_actions_template *at = NULL;
+	uint16_t pos = MLX5_HW_MAX_ACTS;
+	struct rte_flow_action tmp_action[MLX5_HW_MAX_ACTS];
+	struct rte_flow_action tmp_mask[MLX5_HW_MAX_ACTS];
+	const struct rte_flow_action *ra;
+	const struct rte_flow_action *rm;
+	const struct rte_flow_action_modify_field rx_mreg = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = REG_B,
+		},
+		.src = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = REG_C_1,
+		},
+		.width = 32,
+	};
+	const struct rte_flow_action_modify_field rx_mreg_mask = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = UINT32_MAX,
+			.offset = UINT32_MAX,
+		},
+		.src = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = UINT32_MAX,
+			.offset = UINT32_MAX,
+		},
+		.width = UINT32_MAX,
+	};
+	const struct rte_flow_action rx_cpy = {
+		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+		.conf = &rx_mreg,
+	};
+	const struct rte_flow_action rx_cpy_mask = {
+		.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+		.conf = &rx_mreg_mask,
+	};
 
-	if (flow_hw_action_validate(dev, actions, masks, error))
+	if (flow_hw_action_validate(dev, attr, actions, masks, error))
 		return NULL;
-	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS,
-				NULL, 0, actions, error);
+	if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS &&
+	    priv->sh->config.dv_esw_en) {
+		if (flow_hw_action_meta_copy_insert(actions, masks, &rx_cpy, &rx_cpy_mask,
+						    tmp_action, tmp_mask, &pos)) {
+			rte_flow_error_set(error, EINVAL,
+					   RTE_FLOW_ERROR_TYPE_ACTION, NULL,
+					   "Failed to concatenate new action/mask");
+			return NULL;
+		}
+	}
+	/* Application should make sure only one Q/RSS exist in one rule. */
+	if (pos == MLX5_HW_MAX_ACTS) {
+		ra = actions;
+		rm = masks;
+	} else {
+		ra = tmp_action;
+		rm = tmp_mask;
+	}
+	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, ra, error);
 	if (act_len <= 0)
 		return NULL;
 	len = RTE_ALIGN(act_len, 16);
-	mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS,
-				 NULL, 0, masks, error);
+	mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, rm, error);
 	if (mask_len <= 0)
 		return NULL;
 	len += RTE_ALIGN(mask_len, 16);
-	at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at), 64, rte_socket_id());
+	at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at),
+			 RTE_CACHE_LINE_SIZE, rte_socket_id());
 	if (!at) {
 		rte_flow_error_set(error, ENOMEM,
 				   RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
@@ -2469,18 +2719,20 @@  flow_hw_actions_template_create(struct rte_eth_dev *dev,
 				   "cannot allocate action template");
 		return NULL;
 	}
+	/* Actions part is in the first half. */
 	at->attr = *attr;
 	at->actions = (struct rte_flow_action *)(at + 1);
-	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions, len,
-				actions, error);
+	act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions,
+				len, ra, error);
 	if (act_len <= 0)
 		goto error;
-	at->masks = (struct rte_flow_action *)
-		    (((uint8_t *)at->actions) + act_len);
+	/* Masks part is in the second half. */
+	at->masks = (struct rte_flow_action *)(((uint8_t *)at->actions) + act_len);
 	mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->masks,
-				 len - act_len, masks, error);
+				 len - act_len, rm, error);
 	if (mask_len <= 0)
 		goto error;
+	at->rx_cpy_pos = pos;
 	/*
 	 * mlx5 PMD hacks indirect action index directly to the action conf.
 	 * The rte_flow_conv() function copies the content from conf pointer.
@@ -2497,7 +2749,8 @@  flow_hw_actions_template_create(struct rte_eth_dev *dev,
 	LIST_INSERT_HEAD(&priv->flow_hw_at, at, next);
 	return at;
 error:
-	mlx5_free(at);
+	if (at)
+		mlx5_free(at);
 	return NULL;
 }
 
@@ -2572,6 +2825,80 @@  flow_hw_copy_prepend_port_item(const struct rte_flow_item *items,
 	return copied_items;
 }
 
+static int
+flow_hw_pattern_validate(struct rte_eth_dev *dev,
+			 const struct rte_flow_pattern_template_attr *attr,
+			 const struct rte_flow_item items[],
+			 struct rte_flow_error *error)
+{
+	int i;
+	bool items_end = false;
+	RTE_SET_USED(dev);
+	RTE_SET_USED(attr);
+
+	for (i = 0; !items_end; i++) {
+		int type = items[i].type;
+
+		switch (type) {
+		case RTE_FLOW_ITEM_TYPE_TAG:
+		{
+			int reg;
+			const struct rte_flow_item_tag *tag =
+				(const struct rte_flow_item_tag *)items[i].spec;
+
+			reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG, tag->index);
+			if (reg == REG_NON)
+				return rte_flow_error_set(error, EINVAL,
+							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+							  NULL,
+							  "Unsupported tag index");
+			break;
+		}
+		case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
+		{
+			const struct rte_flow_item_tag *tag =
+				(const struct rte_flow_item_tag *)items[i].spec;
+			struct mlx5_priv *priv = dev->data->dev_private;
+			uint8_t regcs = (uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c;
+
+			if (!((1 << (tag->index - REG_C_0)) & regcs))
+				return rte_flow_error_set(error, EINVAL,
+							  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+							  NULL,
+							  "Unsupported internal tag index");
+		}
+		case RTE_FLOW_ITEM_TYPE_VOID:
+		case RTE_FLOW_ITEM_TYPE_ETH:
+		case RTE_FLOW_ITEM_TYPE_VLAN:
+		case RTE_FLOW_ITEM_TYPE_IPV4:
+		case RTE_FLOW_ITEM_TYPE_IPV6:
+		case RTE_FLOW_ITEM_TYPE_UDP:
+		case RTE_FLOW_ITEM_TYPE_TCP:
+		case RTE_FLOW_ITEM_TYPE_GTP:
+		case RTE_FLOW_ITEM_TYPE_GTP_PSC:
+		case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT:
+		case RTE_FLOW_ITEM_TYPE_VXLAN:
+		case MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE:
+		case RTE_FLOW_ITEM_TYPE_META:
+		case RTE_FLOW_ITEM_TYPE_GRE:
+		case RTE_FLOW_ITEM_TYPE_GRE_KEY:
+		case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
+		case RTE_FLOW_ITEM_TYPE_ICMP:
+		case RTE_FLOW_ITEM_TYPE_ICMP6:
+			break;
+		case RTE_FLOW_ITEM_TYPE_END:
+			items_end = true;
+			break;
+		default:
+			return rte_flow_error_set(error, EINVAL,
+						  RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+						  NULL,
+						  "Unsupported item type");
+		}
+	}
+	return 0;
+}
+
 /**
  * Create flow item template.
  *
@@ -2598,6 +2925,8 @@  flow_hw_pattern_template_create(struct rte_eth_dev *dev,
 	struct rte_flow_item *copied_items = NULL;
 	const struct rte_flow_item *tmpl_items;
 
+	if (flow_hw_pattern_validate(dev, attr, items, error))
+		return NULL;
 	if (priv->sh->config.dv_esw_en && attr->ingress) {
 		/*
 		 * Disallow pattern template with ingress and egress/transfer
@@ -3032,6 +3361,17 @@  flow_hw_free_vport_actions(struct mlx5_priv *priv)
 	priv->hw_vport = NULL;
 }
 
+static uint32_t
+flow_hw_usable_lsb_vport_mask(struct mlx5_priv *priv)
+{
+	uint32_t usable_mask = ~priv->vport_meta_mask;
+
+	if (usable_mask)
+		return (1 << rte_bsf32(usable_mask));
+	else
+		return 0;
+}
+
 /**
  * Creates a flow pattern template used to match on E-Switch Manager.
  * This template is used to set up a table for SQ miss default flow.
@@ -3070,7 +3410,10 @@  flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev)
 }
 
 /**
- * Creates a flow pattern template used to match on a TX queue.
+ * Creates a flow pattern template used to match REG_C_0 and a TX queue.
+ * Matching on REG_C_0 is set up to match on least significant bit usable
+ * by user-space, which is set when packet was originated from E-Switch Manager.
+ *
  * This template is used to set up a table for SQ miss default flow.
  *
  * @param dev
@@ -3080,16 +3423,30 @@  flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev)
  *   Pointer to flow pattern template on success, NULL otherwise.
  */
 static struct rte_flow_pattern_template *
-flow_hw_create_ctrl_sq_pattern_template(struct rte_eth_dev *dev)
+flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev)
 {
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t marker_bit = flow_hw_usable_lsb_vport_mask(priv);
 	struct rte_flow_pattern_template_attr attr = {
 		.relaxed_matching = 0,
 		.transfer = 1,
 	};
+	struct rte_flow_item_tag reg_c0_spec = {
+		.index = (uint8_t)REG_C_0,
+	};
+	struct rte_flow_item_tag reg_c0_mask = {
+		.index = 0xff,
+	};
 	struct mlx5_rte_flow_item_tx_queue queue_mask = {
 		.queue = UINT32_MAX,
 	};
 	struct rte_flow_item items[] = {
+		{
+			.type = (enum rte_flow_item_type)
+				MLX5_RTE_FLOW_ITEM_TYPE_TAG,
+			.spec = &reg_c0_spec,
+			.mask = &reg_c0_mask,
+		},
 		{
 			.type = (enum rte_flow_item_type)
 				MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
@@ -3100,6 +3457,12 @@  flow_hw_create_ctrl_sq_pattern_template(struct rte_eth_dev *dev)
 		},
 	};
 
+	if (!marker_bit) {
+		DRV_LOG(ERR, "Unable to set up pattern template for SQ miss table");
+		return NULL;
+	}
+	reg_c0_spec.data = marker_bit;
+	reg_c0_mask.data = marker_bit;
 	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
 }
 
@@ -3137,6 +3500,132 @@  flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev)
 	return flow_hw_pattern_template_create(dev, &attr, items, NULL);
 }
 
+/*
+ * Creating a flow pattern template with all ETH packets matching.
+ * This template is used to set up a table for default Tx copy (Tx metadata
+ * to REG_C_1) flow rule usage.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Pointer to flow pattern template on success, NULL otherwise.
+ */
+static struct rte_flow_pattern_template *
+flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev)
+{
+	struct rte_flow_pattern_template_attr tx_pa_attr = {
+		.relaxed_matching = 0,
+		.egress = 1,
+	};
+	struct rte_flow_item_eth promisc = {
+		.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+		.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+		.type = 0,
+	};
+	struct rte_flow_item eth_all[] = {
+		[0] = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+			.spec = &promisc,
+			.mask = &promisc,
+		},
+		[1] = {
+			.type = RTE_FLOW_ITEM_TYPE_END,
+		},
+	};
+	struct rte_flow_error drop_err;
+
+	RTE_SET_USED(drop_err);
+	return flow_hw_pattern_template_create(dev, &tx_pa_attr, eth_all, &drop_err);
+}
+
+/**
+ * Creates a flow actions template with modify field action and masked jump action.
+ * Modify field action sets the least significant bit of REG_C_0 (usable by user-space)
+ * to 1, meaning that packet was originated from E-Switch Manager. Jump action
+ * transfers steering to group 1.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Pointer to flow actions template on success, NULL otherwise.
+ */
+static struct rte_flow_actions_template *
+flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	uint32_t marker_bit = flow_hw_usable_lsb_vport_mask(priv);
+	uint32_t marker_bit_mask = UINT32_MAX;
+	struct rte_flow_actions_template_attr attr = {
+		.transfer = 1,
+	};
+	struct rte_flow_action_modify_field set_reg_v = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = REG_C_0,
+		},
+		.src = {
+			.field = RTE_FLOW_FIELD_VALUE,
+		},
+		.width = 1,
+	};
+	struct rte_flow_action_modify_field set_reg_m = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = UINT32_MAX,
+			.offset = UINT32_MAX,
+		},
+		.src = {
+			.field = RTE_FLOW_FIELD_VALUE,
+		},
+		.width = UINT32_MAX,
+	};
+	struct rte_flow_action_jump jump_v = {
+		.group = MLX5_HW_LOWEST_USABLE_GROUP,
+	};
+	struct rte_flow_action_jump jump_m = {
+		.group = UINT32_MAX,
+	};
+	struct rte_flow_action actions_v[] = {
+		{
+			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+			.conf = &set_reg_v,
+		},
+		{
+			.type = RTE_FLOW_ACTION_TYPE_JUMP,
+			.conf = &jump_v,
+		},
+		{
+			.type = RTE_FLOW_ACTION_TYPE_END,
+		}
+	};
+	struct rte_flow_action actions_m[] = {
+		{
+			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+			.conf = &set_reg_m,
+		},
+		{
+			.type = RTE_FLOW_ACTION_TYPE_JUMP,
+			.conf = &jump_m,
+		},
+		{
+			.type = RTE_FLOW_ACTION_TYPE_END,
+		}
+	};
+
+	if (!marker_bit) {
+		DRV_LOG(ERR, "Unable to set up actions template for SQ miss table");
+		return NULL;
+	}
+	set_reg_v.dst.offset = rte_bsf32(marker_bit);
+	rte_memcpy(set_reg_v.src.value, &marker_bit, sizeof(marker_bit));
+	rte_memcpy(set_reg_m.src.value, &marker_bit_mask, sizeof(marker_bit_mask));
+	return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, NULL);
+}
+
 /**
  * Creates a flow actions template with an unmasked JUMP action. Flows
  * based on this template will perform a jump to some group. This template
@@ -3231,6 +3720,73 @@  flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev)
 					       NULL);
 }
 
+/*
+ * Creating an actions template to use header modify action for register
+ * copying. This template is used to set up a table for copy flow.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ *
+ * @return
+ *   Pointer to flow actions template on success, NULL otherwise.
+ */
+static struct rte_flow_actions_template *
+flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev)
+{
+	struct rte_flow_actions_template_attr tx_act_attr = {
+		.egress = 1,
+	};
+	const struct rte_flow_action_modify_field mreg_action = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = REG_C_1,
+		},
+		.src = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = REG_A,
+		},
+		.width = 32,
+	};
+	const struct rte_flow_action_modify_field mreg_mask = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = UINT32_MAX,
+			.offset = UINT32_MAX,
+		},
+		.src = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = UINT32_MAX,
+			.offset = UINT32_MAX,
+		},
+		.width = UINT32_MAX,
+	};
+	const struct rte_flow_action copy_reg_action[] = {
+		[0] = {
+			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+			.conf = &mreg_action,
+		},
+		[1] = {
+			.type = RTE_FLOW_ACTION_TYPE_END,
+		},
+	};
+	const struct rte_flow_action copy_reg_mask[] = {
+		[0] = {
+			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+			.conf = &mreg_mask,
+		},
+		[1] = {
+			.type = RTE_FLOW_ACTION_TYPE_END,
+		},
+	};
+	struct rte_flow_error drop_err;
+
+	RTE_SET_USED(drop_err);
+	return flow_hw_actions_template_create(dev, &tx_act_attr, copy_reg_action,
+					       copy_reg_mask, &drop_err);
+}
+
 /**
  * Creates a control flow table used to transfer traffic from E-Switch Manager
  * and TX queues from group 0 to group 1.
@@ -3260,8 +3816,12 @@  flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev,
 		},
 		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
 	};
+	struct mlx5_flow_template_table_cfg cfg = {
+		.attr = attr,
+		.external = false,
+	};
 
-	return flow_hw_table_create(dev, &attr, &it, 1, &at, 1, NULL);
+	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
 }
 
 
@@ -3286,16 +3846,56 @@  flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev,
 {
 	struct rte_flow_template_table_attr attr = {
 		.flow_attr = {
-			.group = MLX5_HW_SQ_MISS_GROUP,
-			.priority = 0,
+			.group = 1,
+			.priority = MLX5_HW_LOWEST_PRIO_NON_ROOT,
 			.ingress = 0,
 			.egress = 0,
 			.transfer = 1,
 		},
 		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
 	};
+	struct mlx5_flow_template_table_cfg cfg = {
+		.attr = attr,
+		.external = false,
+	};
+
+	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
+}
+
+/*
+ * Creating the default Tx metadata copy table on NIC Tx group 0.
+ *
+ * @param dev
+ *   Pointer to Ethernet device.
+ * @param pt
+ *   Pointer to flow pattern template.
+ * @param at
+ *   Pointer to flow actions template.
+ *
+ * @return
+ *   Pointer to flow table on success, NULL otherwise.
+ */
+static struct rte_flow_template_table*
+flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev,
+					  struct rte_flow_pattern_template *pt,
+					  struct rte_flow_actions_template *at)
+{
+	struct rte_flow_template_table_attr tx_tbl_attr = {
+		.flow_attr = {
+			.group = 0, /* Root */
+			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
+			.egress = 1,
+		},
+		.nb_flows = 1, /* One default flow rule for all. */
+	};
+	struct mlx5_flow_template_table_cfg tx_tbl_cfg = {
+		.attr = tx_tbl_attr,
+		.external = false,
+	};
+	struct rte_flow_error drop_err;
 
-	return flow_hw_table_create(dev, &attr, &it, 1, &at, 1, NULL);
+	RTE_SET_USED(drop_err);
+	return flow_hw_table_create(dev, &tx_tbl_cfg, &pt, 1, &at, 1, &drop_err);
 }
 
 /**
@@ -3320,15 +3920,19 @@  flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev,
 	struct rte_flow_template_table_attr attr = {
 		.flow_attr = {
 			.group = 0,
-			.priority = 15, /* TODO: Flow priority discovery. */
+			.priority = MLX5_HW_LOWEST_PRIO_ROOT,
 			.ingress = 0,
 			.egress = 0,
 			.transfer = 1,
 		},
 		.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
 	};
+	struct mlx5_flow_template_table_cfg cfg = {
+		.attr = attr,
+		.external = false,
+	};
 
-	return flow_hw_table_create(dev, &attr, &it, 1, &at, 1, NULL);
+	return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
 }
 
 /**
@@ -3346,11 +3950,14 @@  flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct rte_flow_pattern_template *esw_mgr_items_tmpl = NULL;
-	struct rte_flow_pattern_template *sq_items_tmpl = NULL;
+	struct rte_flow_pattern_template *regc_sq_items_tmpl = NULL;
 	struct rte_flow_pattern_template *port_items_tmpl = NULL;
-	struct rte_flow_actions_template *jump_sq_actions_tmpl = NULL;
+	struct rte_flow_pattern_template *tx_meta_items_tmpl = NULL;
+	struct rte_flow_actions_template *regc_jump_actions_tmpl = NULL;
 	struct rte_flow_actions_template *port_actions_tmpl = NULL;
 	struct rte_flow_actions_template *jump_one_actions_tmpl = NULL;
+	struct rte_flow_actions_template *tx_meta_actions_tmpl = NULL;
+	uint32_t xmeta = priv->sh->config.dv_xmeta_en;
 
 	/* Item templates */
 	esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev);
@@ -3359,8 +3966,8 @@  flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
 			" template for control flows", dev->data->port_id);
 		goto error;
 	}
-	sq_items_tmpl = flow_hw_create_ctrl_sq_pattern_template(dev);
-	if (!sq_items_tmpl) {
+	regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev);
+	if (!regc_sq_items_tmpl) {
 		DRV_LOG(ERR, "port %u failed to create SQ item template for"
 			" control flows", dev->data->port_id);
 		goto error;
@@ -3371,11 +3978,18 @@  flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
 			" control flows", dev->data->port_id);
 		goto error;
 	}
+	if (xmeta == MLX5_XMETA_MODE_META32_HWS) {
+		tx_meta_items_tmpl = flow_hw_create_tx_default_mreg_copy_pattern_template(dev);
+		if (!tx_meta_items_tmpl) {
+			DRV_LOG(ERR, "port %u failed to Tx metadata copy pattern"
+				" template for control flows", dev->data->port_id);
+			goto error;
+		}
+	}
 	/* Action templates */
-	jump_sq_actions_tmpl = flow_hw_create_ctrl_jump_actions_template(dev,
-									 MLX5_HW_SQ_MISS_GROUP);
-	if (!jump_sq_actions_tmpl) {
-		DRV_LOG(ERR, "port %u failed to create jump action template"
+	regc_jump_actions_tmpl = flow_hw_create_ctrl_regc_jump_actions_template(dev);
+	if (!regc_jump_actions_tmpl) {
+		DRV_LOG(ERR, "port %u failed to create REG_C set and jump action template"
 			" for control flows", dev->data->port_id);
 		goto error;
 	}
@@ -3385,23 +3999,32 @@  flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
 			" for control flows", dev->data->port_id);
 		goto error;
 	}
-	jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template(dev, 1);
+	jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template
+			(dev, MLX5_HW_LOWEST_USABLE_GROUP);
 	if (!jump_one_actions_tmpl) {
 		DRV_LOG(ERR, "port %u failed to create jump action template"
 			" for control flows", dev->data->port_id);
 		goto error;
 	}
+	if (xmeta == MLX5_XMETA_MODE_META32_HWS) {
+		tx_meta_actions_tmpl = flow_hw_create_tx_default_mreg_copy_actions_template(dev);
+		if (!tx_meta_actions_tmpl) {
+			DRV_LOG(ERR, "port %u failed to Tx metadata copy actions"
+				" template for control flows", dev->data->port_id);
+			goto error;
+		}
+	}
 	/* Tables */
 	MLX5_ASSERT(priv->hw_esw_sq_miss_root_tbl == NULL);
 	priv->hw_esw_sq_miss_root_tbl = flow_hw_create_ctrl_sq_miss_root_table
-			(dev, esw_mgr_items_tmpl, jump_sq_actions_tmpl);
+			(dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl);
 	if (!priv->hw_esw_sq_miss_root_tbl) {
 		DRV_LOG(ERR, "port %u failed to create table for default sq miss (root table)"
 			" for control flows", dev->data->port_id);
 		goto error;
 	}
 	MLX5_ASSERT(priv->hw_esw_sq_miss_tbl == NULL);
-	priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev, sq_items_tmpl,
+	priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev, regc_sq_items_tmpl,
 								     port_actions_tmpl);
 	if (!priv->hw_esw_sq_miss_tbl) {
 		DRV_LOG(ERR, "port %u failed to create table for default sq miss (non-root table)"
@@ -3416,6 +4039,16 @@  flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
 			" for control flows", dev->data->port_id);
 		goto error;
 	}
+	if (xmeta == MLX5_XMETA_MODE_META32_HWS) {
+		MLX5_ASSERT(priv->hw_tx_meta_cpy_tbl == NULL);
+		priv->hw_tx_meta_cpy_tbl = flow_hw_create_tx_default_mreg_copy_table(dev,
+					tx_meta_items_tmpl, tx_meta_actions_tmpl);
+		if (!priv->hw_tx_meta_cpy_tbl) {
+			DRV_LOG(ERR, "port %u failed to create table for default"
+				" Tx metadata copy flow rule", dev->data->port_id);
+			goto error;
+		}
+	}
 	return 0;
 error:
 	if (priv->hw_esw_zero_tbl) {
@@ -3430,16 +4063,20 @@  flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
 		flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_root_tbl, NULL);
 		priv->hw_esw_sq_miss_root_tbl = NULL;
 	}
+	if (xmeta == MLX5_XMETA_MODE_META32_HWS && tx_meta_actions_tmpl)
+		flow_hw_actions_template_destroy(dev, tx_meta_actions_tmpl, NULL);
 	if (jump_one_actions_tmpl)
 		flow_hw_actions_template_destroy(dev, jump_one_actions_tmpl, NULL);
 	if (port_actions_tmpl)
 		flow_hw_actions_template_destroy(dev, port_actions_tmpl, NULL);
-	if (jump_sq_actions_tmpl)
-		flow_hw_actions_template_destroy(dev, jump_sq_actions_tmpl, NULL);
+	if (regc_jump_actions_tmpl)
+		flow_hw_actions_template_destroy(dev, regc_jump_actions_tmpl, NULL);
+	if (xmeta == MLX5_XMETA_MODE_META32_HWS && tx_meta_items_tmpl)
+		flow_hw_pattern_template_destroy(dev, tx_meta_items_tmpl, NULL);
 	if (port_items_tmpl)
 		flow_hw_pattern_template_destroy(dev, port_items_tmpl, NULL);
-	if (sq_items_tmpl)
-		flow_hw_pattern_template_destroy(dev, sq_items_tmpl, NULL);
+	if (regc_sq_items_tmpl)
+		flow_hw_pattern_template_destroy(dev, regc_sq_items_tmpl, NULL);
 	if (esw_mgr_items_tmpl)
 		flow_hw_pattern_template_destroy(dev, esw_mgr_items_tmpl, NULL);
 	return -EINVAL;
@@ -3491,7 +4128,7 @@  flow_hw_configure(struct rte_eth_dev *dev,
 	struct rte_flow_queue_attr **_queue_attr = NULL;
 	struct rte_flow_queue_attr ctrl_queue_attr = {0};
 	bool is_proxy = !!(priv->sh->config.dv_esw_en && priv->master);
-	int ret;
+	int ret = 0;
 
 	if (!port_attr || !nb_queue || !queue_attr) {
 		rte_errno = EINVAL;
@@ -3642,6 +4279,9 @@  flow_hw_configure(struct rte_eth_dev *dev,
 	}
 	if (_queue_attr)
 		mlx5_free(_queue_attr);
+	/* Do not overwrite the internal errno information. */
+	if (ret)
+		return ret;
 	return rte_flow_error_set(error, rte_errno,
 				  RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 				  "fail to configure port");
@@ -3751,17 +4391,17 @@  void flow_hw_init_tags_set(struct rte_eth_dev *dev)
 		return;
 	unset |= 1 << (priv->mtr_color_reg - REG_C_0);
 	unset |= 1 << (REG_C_6 - REG_C_0);
-	if (meta_mode == MLX5_XMETA_MODE_META32_HWS) {
-		unset |= 1 << (REG_C_1 - REG_C_0);
+	if (priv->sh->config.dv_esw_en)
 		unset |= 1 << (REG_C_0 - REG_C_0);
-	}
+	if (meta_mode == MLX5_XMETA_MODE_META32_HWS)
+		unset |= 1 << (REG_C_1 - REG_C_0);
 	masks &= ~unset;
 	if (mlx5_flow_hw_avl_tags_init_cnt) {
 		for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) {
 			if (mlx5_flow_hw_avl_tags[i] != REG_NON && !!((1 << i) & masks)) {
 				copy[mlx5_flow_hw_avl_tags[i] - REG_C_0] =
 						mlx5_flow_hw_avl_tags[i];
-				copy_masks |= (1 << i);
+				copy_masks |= (1 << (mlx5_flow_hw_avl_tags[i] - REG_C_0));
 			}
 		}
 		if (copy_masks != masks) {
@@ -3903,7 +4543,6 @@  flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
 	return flow_dv_action_destroy(dev, handle, error);
 }
 
-
 const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
 	.info_get = flow_hw_info_get,
 	.configure = flow_hw_configure,
@@ -3911,7 +4550,7 @@  const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
 	.pattern_template_destroy = flow_hw_pattern_template_destroy,
 	.actions_template_create = flow_hw_actions_template_create,
 	.actions_template_destroy = flow_hw_actions_template_destroy,
-	.template_table_create = flow_hw_table_create,
+	.template_table_create = flow_hw_template_table_create,
 	.template_table_destroy = flow_hw_table_destroy,
 	.async_flow_create = flow_hw_async_flow_create,
 	.async_flow_destroy = flow_hw_async_flow_destroy,
@@ -3927,13 +4566,6 @@  const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
 	.action_query = flow_dv_action_query,
 };
 
-static uint32_t
-flow_hw_get_ctrl_queue(struct mlx5_priv *priv)
-{
-	MLX5_ASSERT(priv->nb_queue > 0);
-	return priv->nb_queue - 1;
-}
-
 /**
  * Creates a control flow using flow template API on @p proxy_dev device,
  * on behalf of @p owner_dev device.
@@ -3971,7 +4603,7 @@  flow_hw_create_ctrl_flow(struct rte_eth_dev *owner_dev,
 			 uint8_t action_template_idx)
 {
 	struct mlx5_priv *priv = proxy_dev->data->dev_private;
-	uint32_t queue = flow_hw_get_ctrl_queue(priv);
+	uint32_t queue = priv->nb_queue - 1;
 	struct rte_flow_op_attr op_attr = {
 		.postpone = 0,
 	};
@@ -4046,7 +4678,7 @@  static int
 flow_hw_destroy_ctrl_flow(struct rte_eth_dev *dev, struct rte_flow *flow)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
-	uint32_t queue = flow_hw_get_ctrl_queue(priv);
+	uint32_t queue = priv->nb_queue - 1;
 	struct rte_flow_op_attr op_attr = {
 		.postpone = 0,
 	};
@@ -4183,10 +4815,24 @@  mlx5_flow_hw_esw_create_mgr_sq_miss_flow(struct rte_eth_dev *dev)
 			.type = RTE_FLOW_ITEM_TYPE_END,
 		},
 	};
+	struct rte_flow_action_modify_field modify_field = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+		},
+		.src = {
+			.field = RTE_FLOW_FIELD_VALUE,
+		},
+		.width = 1,
+	};
 	struct rte_flow_action_jump jump = {
-		.group = MLX5_HW_SQ_MISS_GROUP,
+		.group = 1,
 	};
 	struct rte_flow_action actions[] = {
+		{
+			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+			.conf = &modify_field,
+		},
 		{
 			.type = RTE_FLOW_ACTION_TYPE_JUMP,
 			.conf = &jump,
@@ -4209,6 +4855,12 @@  int
 mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
 {
 	uint16_t port_id = dev->data->port_id;
+	struct rte_flow_item_tag reg_c0_spec = {
+		.index = (uint8_t)REG_C_0,
+	};
+	struct rte_flow_item_tag reg_c0_mask = {
+		.index = 0xff,
+	};
 	struct mlx5_rte_flow_item_tx_queue queue_spec = {
 		.queue = txq,
 	};
@@ -4216,6 +4868,12 @@  mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
 		.queue = UINT32_MAX,
 	};
 	struct rte_flow_item items[] = {
+		{
+			.type = (enum rte_flow_item_type)
+				MLX5_RTE_FLOW_ITEM_TYPE_TAG,
+			.spec = &reg_c0_spec,
+			.mask = &reg_c0_mask,
+		},
 		{
 			.type = (enum rte_flow_item_type)
 				MLX5_RTE_FLOW_ITEM_TYPE_TX_QUEUE,
@@ -4241,6 +4899,7 @@  mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
 	struct rte_eth_dev *proxy_dev;
 	struct mlx5_priv *proxy_priv;
 	uint16_t proxy_port_id = dev->data->port_id;
+	uint32_t marker_bit;
 	int ret;
 
 	RTE_SET_USED(txq);
@@ -4261,6 +4920,14 @@  mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t txq)
 		rte_errno = ENOMEM;
 		return -rte_errno;
 	}
+	marker_bit = flow_hw_usable_lsb_vport_mask(proxy_priv);
+	if (!marker_bit) {
+		DRV_LOG(ERR, "Unable to set up control flow in SQ miss table");
+		rte_errno = EINVAL;
+		return -rte_errno;
+	}
+	reg_c0_spec.data = marker_bit;
+	reg_c0_mask.data = marker_bit;
 	return flow_hw_create_ctrl_flow(dev, proxy_dev,
 					proxy_priv->hw_esw_sq_miss_tbl,
 					items, 0, actions, 0);
@@ -4320,4 +4987,53 @@  mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev)
 					items, 0, actions, 0);
 }
 
+int
+mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct rte_flow_item_eth promisc = {
+		.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+		.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
+		.type = 0,
+	};
+	struct rte_flow_item eth_all[] = {
+		[0] = {
+			.type = RTE_FLOW_ITEM_TYPE_ETH,
+			.spec = &promisc,
+			.mask = &promisc,
+		},
+		[1] = {
+			.type = RTE_FLOW_ITEM_TYPE_END,
+		},
+	};
+	struct rte_flow_action_modify_field mreg_action = {
+		.operation = RTE_FLOW_MODIFY_SET,
+		.dst = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = REG_C_1,
+		},
+		.src = {
+			.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
+			.level = REG_A,
+		},
+		.width = 32,
+	};
+	struct rte_flow_action copy_reg_action[] = {
+		[0] = {
+			.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
+			.conf = &mreg_action,
+		},
+		[1] = {
+			.type = RTE_FLOW_ACTION_TYPE_END,
+		},
+	};
+
+	MLX5_ASSERT(priv->master);
+	if (!priv->dr_ctx || !priv->hw_tx_meta_cpy_tbl)
+		return 0;
+	return flow_hw_create_ctrl_flow(dev, dev,
+					priv->hw_tx_meta_cpy_tbl,
+					eth_all, 0, copy_reg_action, 0);
+}
+
 #endif
diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c
index 6313602a66..ccefebefc9 100644
--- a/drivers/net/mlx5/mlx5_trigger.c
+++ b/drivers/net/mlx5/mlx5_trigger.c
@@ -1292,6 +1292,9 @@  mlx5_traffic_enable_hws(struct rte_eth_dev *dev)
 	if (priv->sh->config.dv_esw_en && priv->master) {
 		if (mlx5_flow_hw_esw_create_mgr_sq_miss_flow(dev))
 			goto error;
+		if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS)
+			if (mlx5_flow_hw_create_tx_default_mreg_copy_flow(dev))
+				goto error;
 	}
 	for (i = 0; i < priv->txqs_n; ++i) {
 		struct mlx5_txq_ctrl *txq = mlx5_txq_get(dev, i);