diff mbox

[dpdk-dev,10/18] net/mlx5: support packet tunnel type

Message ID 20180226150947.107179-11-xuemingl@mellanox.com (mailing list archive)
State Superseded, archived
Headers show

Checks

Context Check Description
ci/checkpatch success coding style OK
ci/Intel-compilation success Compilation OK

Commit Message

Xueming Li Feb. 26, 2018, 3:09 p.m. UTC
Support packet tunnel type identification based on rte_flow rule.

If rules of multiple tunnel types specified on same queue,
RTE_PTYPE_TUNNEL_MASK will be returned as tunnel type, it's user's
responsibility to set bits in flow mark as tunnel type identifier.

Signed-off-by: Xueming Li <xuemingl@mellanox.com>
---
 drivers/net/mlx5/mlx5_flow.c          | 72 +++++++++++++++++++++++++++++++----
 drivers/net/mlx5/mlx5_rxq.c           | 13 ++++++-
 drivers/net/mlx5/mlx5_rxtx.c          | 12 ++++--
 drivers/net/mlx5/mlx5_rxtx.h          |  7 +++-
 drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 21 ++++++----
 drivers/net/mlx5/mlx5_rxtx_vec_sse.h  | 17 +++++++--
 6 files changed, 115 insertions(+), 27 deletions(-)
diff mbox

Patch

diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c
index 755355e..0f59de6 100644
--- a/drivers/net/mlx5/mlx5_flow.c
+++ b/drivers/net/mlx5/mlx5_flow.c
@@ -215,6 +215,7 @@  struct rte_flow {
 	uint16_t (*queues)[]; /**< Queues indexes to use. */
 	struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
 	uint8_t rss_key[40]; /**< copy of the RSS key. */
+	uint32_t tunnel; /* Tunnel type as RTE_PTYPE_TUNNEL_XXX. */
 	struct ibv_counter_set *cs; /**< Holds the counters for the rule. */
 	struct mlx5_flow_counter_stats counter_stats;/**<The counter stats. */
 	struct mlx5_flow frxq[RTE_DIM(hash_rxq_init)];
@@ -231,6 +232,18 @@  struct rte_flow {
 	(type) == RTE_FLOW_ITEM_TYPE_VXLAN || \
 	(type) == RTE_FLOW_ITEM_TYPE_GRE)
 
+const uint32_t rte_ptype[] = {
+	[RTE_FLOW_ITEM_TYPE_VXLAN]   = RTE_PTYPE_TUNNEL_VXLAN |
+				       RTE_PTYPE_L4_UDP,
+	[RTE_FLOW_ITEM_TYPE_GRE]     = RTE_PTYPE_TUNNEL_GRE,
+};
+
+#define PTYPE_TUN(t) ((RTE_PTYPE_TUNNEL_MASK & (t)) >> 12)
+static const uint32_t ptype_flow_type[] = {
+	[PTYPE_TUN(RTE_PTYPE_TUNNEL_VXLAN)]   = RTE_FLOW_ITEM_TYPE_VXLAN,
+	[PTYPE_TUN(RTE_PTYPE_TUNNEL_GRE)]     = RTE_FLOW_ITEM_TYPE_GRE,
+};
+
 /** Structure to generate a simple graph of layers supported by the NIC. */
 struct mlx5_flow_items {
 	/** List of possible actions for these items. */
@@ -428,6 +441,7 @@  struct mlx5_flow_parse {
 	struct rte_eth_rss_conf rss_conf; /**< RSS configuration */
 	uint8_t rss_key[40]; /**< copy of the RSS key. */
 	enum hash_rxq_type layer; /**< Last pattern layer detected. */
+	uint32_t tunnel; /* Tunnel type as RTE_PTYPE_TUNNEL_XXX. */
 	struct ibv_counter_set *cs; /**< Holds the counter set for the rule */
 	struct {
 		struct ibv_flow_attr *ibv_attr;
@@ -849,7 +863,7 @@  struct ibv_spec_header {
 		if (err)
 			goto exit_item_not_supported;
 		if (IS_TUNNEL(items->type)) {
-			if (parser->inner) {
+			if (parser->tunnel) {
 				rte_flow_error_set(error, ENOTSUP,
 						   RTE_FLOW_ERROR_TYPE_ITEM,
 						   items,
@@ -864,6 +878,7 @@  struct ibv_spec_header {
 					"Cannot support hw tunnel checksum "
 					"offloads");
 			parser->inner = IBV_FLOW_SPEC_INNER;
+			parser->tunnel = rte_ptype[items->type];
 		}
 		if (parser->drop) {
 			parser->queue[HASH_RXQ_ETH].offset += cur_item->dst_sz;
@@ -1132,6 +1147,7 @@  struct ibv_spec_header {
 	}
 	/* Third step. Conversion parse, fill the specifications. */
 	parser->inner = 0;
+	parser->tunnel = 0;
 	for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
 		if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
 			continue;
@@ -1562,6 +1578,7 @@  struct ibv_spec_header {
 
 	id.vni[0] = 0;
 	parser->inner = IBV_FLOW_SPEC_INNER;
+	parser->tunnel = rte_ptype[item->type];
 	if (spec) {
 		if (!mask)
 			mask = default_mask;
@@ -1611,6 +1628,7 @@  struct ibv_spec_header {
 	(void)item;
 	(void)default_mask;
 	parser->inner = IBV_FLOW_SPEC_INNER;
+	parser->tunnel = rte_ptype[item->type];
 	mlx5_flow_create_copy(parser, &tunnel, size);
 	return 0;
 }
@@ -1783,7 +1801,8 @@  struct ibv_spec_header {
 					   parser->rss_conf.rss_key_len,
 					   hash_fields,
 					   parser->queues,
-					   parser->queues_n);
+					   parser->queues_n,
+					   parser->tunnel);
 		if (flow->frxq[i].hrxq)
 			continue;
 		flow->frxq[i].hrxq =
@@ -1792,7 +1811,8 @@  struct ibv_spec_header {
 					   parser->rss_conf.rss_key_len,
 					   hash_fields,
 					   parser->queues,
-					   parser->queues_n);
+					   parser->queues_n,
+					   parser->tunnel);
 		if (!flow->frxq[i].hrxq) {
 			rte_flow_error_set(error, ENOMEM,
 					   RTE_FLOW_ERROR_TYPE_HANDLE,
@@ -1858,8 +1878,16 @@  struct ibv_spec_header {
 	for (i = 0; i != parser->queues_n; ++i) {
 		struct mlx5_rxq_data *q =
 			(*priv->rxqs)[parser->queues[i]];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of(q, struct mlx5_rxq_ctrl, rxq);
+		uint8_t tunnel = PTYPE_TUN(parser->tunnel);
 
 		q->mark |= parser->mark;
+		if (parser->tunnel)
+			rxq_ctrl->tunnel_types[tunnel] += 1;
+		if (q->tunnel != parser->tunnel)
+			q->tunnel = q->tunnel ? RTE_PTYPE_TUNNEL_MASK :
+				    parser->tunnel;
 	}
 	return 0;
 error:
@@ -1932,6 +1960,7 @@  struct ibv_spec_header {
 	flow->queues = (uint16_t (*)[])(flow + 1);
 	memcpy(flow->queues, parser.queues, parser.queues_n * sizeof(uint16_t));
 	flow->queues_n = parser.queues_n;
+	flow->tunnel = parser.tunnel;
 	flow->mark = parser.mark;
 	/* Copy RSS configuration. */
 	flow->rss_conf = parser.rss_conf;
@@ -2021,9 +2050,36 @@  struct rte_flow *
 {
 	unsigned int i;
 
-	if (flow->drop || !flow->mark)
+	if (flow->drop)
 		goto free;
-	for (i = 0; i != flow->queues_n; ++i) {
+	for (i = 0; flow->tunnel && i != flow->queues_n; ++i) {
+		/* Update queue tunnel type. */
+		struct mlx5_rxq_data *rxq = (*priv->rxqs)[(*flow->queues)[i]];
+		struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+		uint8_t tunnel = PTYPE_TUN(flow->tunnel);
+
+		RTE_ASSERT(rxq_ctrl->tunnel_types[tunnel] > 0);
+		rxq_ctrl->tunnel_types[tunnel] -= 1;
+		if (!rxq_ctrl->tunnel_types[tunnel]) {
+			/* Update tunnel type. */
+			uint8_t j;
+			uint8_t types = 0;
+			uint8_t last;
+
+			for (j = 0; j < RTE_DIM(rxq_ctrl->tunnel_types); j++)
+				if (rxq_ctrl->tunnel_types[j]) {
+					types += 1;
+					last = j;
+				}
+			/* Keep same if more than one tunnel types left. */
+			if (types == 1)
+				rxq->tunnel = rte_ptype[ptype_flow_type[last]];
+			else if (types == 0) /* No tunnel type left. */
+				rxq->tunnel = 0;
+		}
+	}
+	for (i = 0; flow->mark && i != flow->queues_n; ++i) {
 		struct rte_flow *tmp;
 		int mark = 0;
 
@@ -2315,7 +2371,8 @@  struct rte_flow *
 						   flow->rss_conf.rss_key_len,
 						   hash_rxq_init[i].hash_fields,
 						   (*flow->queues),
-						   flow->queues_n);
+						   flow->queues_n,
+						   flow->tunnel);
 			if (flow->frxq[i].hrxq)
 				goto flow_create;
 			flow->frxq[i].hrxq =
@@ -2323,7 +2380,8 @@  struct rte_flow *
 						   flow->rss_conf.rss_key_len,
 						   hash_rxq_init[i].hash_fields,
 						   (*flow->queues),
-						   flow->queues_n);
+						   flow->queues_n,
+						   flow->tunnel);
 			if (!flow->frxq[i].hrxq) {
 				DEBUG("Flow %p cannot be applied",
 				      (void *)flow);
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index 9bc8b30..09ef189 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -1325,13 +1325,16 @@  struct mlx5_ind_table_ibv*
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
+ * @param tunnel
+ *   Tunnel type.
  *
  * @return
  *   An hash Rx queue on success.
  */
 struct mlx5_hrxq*
 mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
-		   uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
+		   uint64_t hash_fields, uint16_t queues[], uint16_t queues_n,
+		   uint32_t tunnel)
 {
 	struct mlx5_hrxq *hrxq;
 	struct mlx5_ind_table_ibv *ind_tbl;
@@ -1369,6 +1372,7 @@  struct mlx5_hrxq*
 	hrxq->qp = qp;
 	hrxq->rss_key_len = rss_key_len;
 	hrxq->hash_fields = hash_fields;
+	hrxq->tunnel = tunnel;
 	memcpy(hrxq->rss_key, rss_key, rss_key_len);
 	rte_atomic32_inc(&hrxq->refcnt);
 	LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
@@ -1394,13 +1398,16 @@  struct mlx5_hrxq*
  *   first queue index will be taken for the indirection table.
  * @param queues_n
  *   Number of queues.
+ * @param tunnel
+ *   Tunnel type.
  *
  * @return
  *   An hash Rx queue on success.
  */
 struct mlx5_hrxq*
 mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
-		   uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
+		   uint64_t hash_fields, uint16_t queues[], uint16_t queues_n,
+		   uint32_t tunnel)
 {
 	struct mlx5_hrxq *hrxq;
 
@@ -1414,6 +1421,8 @@  struct mlx5_hrxq*
 			continue;
 		if (hrxq->hash_fields != hash_fields)
 			continue;
+		if (hrxq->tunnel != tunnel)
+			continue;
 		ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n);
 		if (!ind_tbl)
 			continue;
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index d9bfc9b..926e11d 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -34,7 +34,7 @@ 
 #include "mlx5_prm.h"
 
 static __rte_always_inline uint32_t
-rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe);
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe);
 
 static __rte_always_inline int
 mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe,
@@ -109,12 +109,14 @@ 
 	(*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_L4_UDP;
 	/* Tunneled - L3 */
+	(*p)[0x40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
 	(*p)[0x41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_NONFRAG;
 	(*p)[0x42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_NONFRAG;
+	(*p)[0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
 	(*p)[0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 		     RTE_PTYPE_INNER_L4_NONFRAG;
@@ -1541,6 +1543,8 @@ 
 /**
  * Translate RX completion flags to packet type.
  *
+ * @param[in] rxq
+ *   Pointer to RX queue structure.
  * @param[in] cqe
  *   Pointer to CQE.
  *
@@ -1550,7 +1554,7 @@ 
  *   Packet type for struct rte_mbuf.
  */
 static inline uint32_t
-rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe)
+rxq_cq_to_pkt_type(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe)
 {
 	uint8_t idx;
 	uint8_t pinfo = cqe->pkt_info;
@@ -1565,7 +1569,7 @@ 
 	 * bit[7] = outer_l3_type
 	 */
 	idx = ((pinfo & 0x3) << 6) | ((ptype & 0xfc00) >> 10);
-	return mlx5_ptype_table[idx];
+	return mlx5_ptype_table[idx] | rxq->tunnel * !!(idx & (1 << 6));
 }
 
 /**
@@ -1787,7 +1791,7 @@ 
 			pkt = seg;
 			assert(len >= (rxq->crc_present << 2));
 			/* Update packet information. */
-			pkt->packet_type = rxq_cq_to_pkt_type(cqe);
+			pkt->packet_type = rxq_cq_to_pkt_type(rxq, cqe);
 			pkt->ol_flags = 0;
 			if (rss_hash_res && rxq->rss_hash) {
 				pkt->hash.rss = rss_hash_res;
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 7fce001..7763641 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -103,6 +103,7 @@  struct mlx5_rxq_data {
 	void *cq_uar; /* CQ user access region. */
 	uint32_t cqn; /* CQ number. */
 	uint8_t cq_arm_sn; /* CQ arm seq number. */
+	uint32_t tunnel; /* Tunnel type. */
 } __rte_cache_aligned;
 
 /* Verbs Rx queue elements. */
@@ -124,6 +125,7 @@  struct mlx5_rxq_ctrl {
 	struct mlx5_rxq_ibv *ibv; /* Verbs elements. */
 	struct mlx5_rxq_data rxq; /* Data path structure. */
 	unsigned int socket; /* CPU socket ID for allocations. */
+	uint32_t tunnel_types[16]; /* Tunnel type counter. */
 	unsigned int irq:1; /* Whether IRQ is enabled. */
 };
 
@@ -143,6 +145,7 @@  struct mlx5_hrxq {
 	struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */
 	struct ibv_qp *qp; /* Verbs queue pair. */
 	uint64_t hash_fields; /* Verbs Hash fields. */
+	uint32_t tunnel; /* Tunnel type. */
 	uint8_t rss_key_len; /* Hash key length in bytes. */
 	uint8_t rss_key[]; /* Hash key. */
 };
@@ -240,9 +243,9 @@  struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_get(struct priv *,
 int mlx5_priv_ind_table_ibv_release(struct priv *, struct mlx5_ind_table_ibv *);
 int mlx5_priv_ind_table_ibv_verify(struct priv *);
 struct mlx5_hrxq *mlx5_priv_hrxq_new(struct priv *, uint8_t *, uint8_t,
-				     uint64_t, uint16_t [], uint16_t);
+				     uint64_t, uint16_t [], uint16_t, uint32_t);
 struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *, uint8_t *, uint8_t,
-				     uint64_t, uint16_t [], uint16_t);
+				     uint64_t, uint16_t [], uint16_t, uint32_t);
 int mlx5_priv_hrxq_release(struct priv *, struct mlx5_hrxq *);
 int mlx5_priv_hrxq_ibv_verify(struct priv *);
 uint64_t mlx5_priv_get_rx_port_offloads(struct priv *);
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
index bbe1818..9f91361 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_neon.h
@@ -551,6 +551,7 @@ 
 	const uint64x1_t mbuf_init = vld1_u64(&rxq->mbuf_initializer);
 	const uint64x1_t r32_mask = vcreate_u64(0xffffffff);
 	uint64x2_t rearm0, rearm1, rearm2, rearm3;
+	uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
 
 	if (rxq->mark) {
 		const uint32x4_t ft_def = vdupq_n_u32(MLX5_FLOW_MARK_DEFAULT);
@@ -583,14 +584,18 @@ 
 	ptype = vshrn_n_u32(ptype_info, 10);
 	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
 	ptype = vorr_u16(ptype, op_err);
-	pkts[0]->packet_type =
-		mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 6)];
-	pkts[1]->packet_type =
-		mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 4)];
-	pkts[2]->packet_type =
-		mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 2)];
-	pkts[3]->packet_type =
-		mlx5_ptype_table[vget_lane_u8(vreinterpret_u8_u16(ptype), 0)];
+	pt_idx0 = vget_lane_u8(vreinterpret_u8_u16(ptype), 6);
+	pt_idx1 = vget_lane_u8(vreinterpret_u8_u16(ptype), 4);
+	pt_idx2 = vget_lane_u8(vreinterpret_u8_u16(ptype), 2);
+	pt_idx3 = vget_lane_u8(vreinterpret_u8_u16(ptype), 0);
+	pkts[0]->packet_type = mlx5_ptype_table[pt_idx0] |
+			       !!(pt_idx0 & (1 << 6)) * rxq->tunnel;
+	pkts[1]->packet_type = mlx5_ptype_table[pt_idx1] |
+			       !!(pt_idx1 & (1 << 6)) * rxq->tunnel;
+	pkts[2]->packet_type = mlx5_ptype_table[pt_idx2] |
+			       !!(pt_idx2 & (1 << 6)) * rxq->tunnel;
+	pkts[3]->packet_type = mlx5_ptype_table[pt_idx3] |
+			       !!(pt_idx3 & (1 << 6)) * rxq->tunnel;
 	/* Fill flags for checksum and VLAN. */
 	pinfo = vandq_u32(ptype_info, ptype_ol_mask);
 	pinfo = vreinterpretq_u32_u8(
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index c088bcb..d249248 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -542,6 +542,7 @@ 
 	const __m128i mbuf_init =
 		_mm_loadl_epi64((__m128i *)&rxq->mbuf_initializer);
 	__m128i rearm0, rearm1, rearm2, rearm3;
+	uint8_t pt_idx0, pt_idx1, pt_idx2, pt_idx3;
 
 	/* Extract pkt_info field. */
 	pinfo0 = _mm_unpacklo_epi32(cqes[0], cqes[1]);
@@ -595,10 +596,18 @@ 
 	/* Errored packets will have RTE_PTYPE_ALL_MASK. */
 	op_err = _mm_srli_epi16(op_err, 8);
 	ptype = _mm_or_si128(ptype, op_err);
-	pkts[0]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 0)];
-	pkts[1]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 2)];
-	pkts[2]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 4)];
-	pkts[3]->packet_type = mlx5_ptype_table[_mm_extract_epi8(ptype, 6)];
+	pt_idx0 = _mm_extract_epi8(ptype, 0);
+	pt_idx1 = _mm_extract_epi8(ptype, 2);
+	pt_idx2 = _mm_extract_epi8(ptype, 4);
+	pt_idx3 = _mm_extract_epi8(ptype, 6);
+	pkts[0]->packet_type = mlx5_ptype_table[pt_idx0] |
+			       !!(pt_idx0 & (1 << 6)) * rxq->tunnel;
+	pkts[1]->packet_type = mlx5_ptype_table[pt_idx1] |
+			       !!(pt_idx1 & (1 << 6)) * rxq->tunnel;
+	pkts[2]->packet_type = mlx5_ptype_table[pt_idx2] |
+			       !!(pt_idx2 & (1 << 6)) * rxq->tunnel;
+	pkts[3]->packet_type = mlx5_ptype_table[pt_idx3] |
+			       !!(pt_idx3 & (1 << 6)) * rxq->tunnel;
 	/* Fill flags for checksum and VLAN. */
 	pinfo = _mm_and_si128(pinfo, ptype_ol_mask);
 	pinfo = _mm_shuffle_epi8(cv_flag_sel, pinfo);