net/iavf: fix avx512 Tx path
Checks
Commit Message
This patch fixes the tunneling parameters on avx512 Tx path and
adjusts the path selection to fix perf drop.
Fixes: 4f8259df563a ("net/iavf: enable Tx outer checksum offload on AVX512")
Signed-off-by: Zhichao Zeng <zhichaox.zeng@intel.com>
---
drivers/net/iavf/iavf_rxtx.c | 7 +--
drivers/net/iavf/iavf_rxtx.h | 5 +-
drivers/net/iavf/iavf_rxtx_vec_avx512.c | 82 ++++++++++++++++++++++---
drivers/net/iavf/iavf_rxtx_vec_common.h | 23 +++----
4 files changed, 87 insertions(+), 30 deletions(-)
Comments
> -----Original Message-----
> From: Zeng, ZhichaoX <zhichaox.zeng@intel.com>
> Sent: Wednesday, February 22, 2023 4:13 PM
> To: dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xu, Ke1 <ke1.xu@intel.com>; Chu,
> Haijun <haijun.chu@intel.com>; Zeng, ZhichaoX <zhichaox.zeng@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>; Xing, Beilei <beilei.xing@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>; Konstantin Ananyev
> <konstantin.v.ananyev@yandex.ru>; Zhou, YidingX <yidingx.zhou@intel.com>
> Subject: [PATCH] net/iavf: fix avx512 Tx path
>
> This patch fixes the tunneling parameters on avx512 Tx path and adjusts the
> path selection to fix perf drop.
>
> Fixes: 4f8259df563a ("net/iavf: enable Tx outer checksum offload on
> AVX512")
>
> Signed-off-by: Zhichao Zeng <zhichaox.zeng@intel.com>
Validated function and perf, passed.
Tested-by: Ke Xu <ke1.xu@intel.com>
> ---
> drivers/net/iavf/iavf_rxtx.c | 7 +--
> drivers/net/iavf/iavf_rxtx.h | 5 +-
> drivers/net/iavf/iavf_rxtx_vec_avx512.c | 82 ++++++++++++++++++++++---
> drivers/net/iavf/iavf_rxtx_vec_common.h | 23 +++----
> 4 files changed, 87 insertions(+), 30 deletions(-)
>
> -----Original Message-----
> From: Xu, Ke1 <ke1.xu@intel.com>
> Sent: Thursday, February 23, 2023 4:17 PM
> To: Zeng, ZhichaoX <zhichaox.zeng@intel.com>; dev@dpdk.org
> Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Chu, Haijun <haijun.chu@intel.com>;
> Wu, Jingjing <jingjing.wu@intel.com>; Xing, Beilei <beilei.xing@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>; Konstantin Ananyev
> <konstantin.v.ananyev@yandex.ru>; Zhou, YidingX <yidingx.zhou@intel.com>
> Subject: RE: [PATCH] net/iavf: fix avx512 Tx path
>
>
> > -----Original Message-----
> > From: Zeng, ZhichaoX <zhichaox.zeng@intel.com>
> > Sent: Wednesday, February 22, 2023 4:13 PM
> > To: dev@dpdk.org
> > Cc: Zhang, Qi Z <qi.z.zhang@intel.com>; Xu, Ke1 <ke1.xu@intel.com>;
> > Chu, Haijun <haijun.chu@intel.com>; Zeng, ZhichaoX
> > <zhichaox.zeng@intel.com>; Wu, Jingjing <jingjing.wu@intel.com>; Xing,
> > Beilei <beilei.xing@intel.com>; Richardson, Bruce
> > <bruce.richardson@intel.com>; Konstantin Ananyev
> > <konstantin.v.ananyev@yandex.ru>; Zhou, YidingX
> > <yidingx.zhou@intel.com>
> > Subject: [PATCH] net/iavf: fix avx512 Tx path
> >
> > This patch fixes the tunneling parameters on avx512 Tx path and
> > adjusts the path selection to fix perf drop.
> >
> > Fixes: 4f8259df563a ("net/iavf: enable Tx outer checksum offload on
> > AVX512")
> >
> > Signed-off-by: Zhichao Zeng <zhichaox.zeng@intel.com>
>
> Validated function and perf, passed.
> Tested-by: Ke Xu <ke1.xu@intel.com>
>
> > ---
> > drivers/net/iavf/iavf_rxtx.c | 7 +--
> > drivers/net/iavf/iavf_rxtx.h | 5 +-
> > drivers/net/iavf/iavf_rxtx_vec_avx512.c | 82
> > ++++++++++++++++++++++--- drivers/net/iavf/iavf_rxtx_vec_common.h |
> 23
> > +++----
> > 4 files changed, 87 insertions(+), 30 deletions(-)
> >
Applied to dpdk-next-net-intel.
Thanks
Qi
@@ -3877,7 +3877,7 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
if (check_ret >= 0 &&
rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
/* SSE and AVX2 not support offload path yet. */
- if (check_ret == IAVF_VECTOR_PATH || check_ret == IAVF_VECTOR_CTX_PATH) {
+ if (check_ret == IAVF_VECTOR_PATH) {
use_sse = true;
if ((rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1 ||
rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) == 1) &&
@@ -3914,11 +3914,6 @@ iavf_set_tx_function(struct rte_eth_dev *dev)
dev->tx_pkt_prepare = iavf_prep_pkts;
PMD_DRV_LOG(DEBUG, "Using AVX512 OFFLOAD Vector Tx (port %d).",
dev->data->port_id);
- } else if (check_ret == IAVF_VECTOR_CTX_PATH) {
- dev->tx_pkt_burst = iavf_xmit_pkts_vec_avx512_ctx;
- dev->tx_pkt_prepare = iavf_prep_pkts;
- PMD_DRV_LOG(DEBUG, "Using AVX512 CONTEXT Vector Tx (port %d).",
- dev->data->port_id);
} else {
dev->tx_pkt_burst = iavf_xmit_pkts_vec_avx512_ctx_offload;
dev->tx_pkt_prepare = iavf_prep_pkts;
@@ -57,8 +57,7 @@
#define IAVF_VECTOR_PATH 0
#define IAVF_VECTOR_OFFLOAD_PATH 1
-#define IAVF_VECTOR_CTX_PATH 2
-#define IAVF_VECTOR_CTX_OFFLOAD_PATH 3
+#define IAVF_VECTOR_CTX_OFFLOAD_PATH 2
#define DEFAULT_TX_RS_THRESH 32
#define DEFAULT_TX_FREE_THRESH 32
@@ -727,8 +726,6 @@ uint16_t iavf_xmit_pkts_vec_avx512(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t iavf_xmit_pkts_vec_avx512_offload(void *tx_queue,
struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
-uint16_t iavf_xmit_pkts_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts,
- uint16_t nb_pkts);
uint16_t iavf_xmit_pkts_vec_avx512_ctx_offload(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
int iavf_txq_vec_setup_avx512(struct iavf_tx_queue *txq);
@@ -1916,6 +1916,79 @@ iavf_fill_ctx_desc_tunneling_avx512(uint64_t *low_ctx_qw, struct rte_mbuf *pkt)
}
}
+static inline void
+iavf_fill_ctx_desc_tunnelling_field(volatile uint64_t *qw0,
+ const struct rte_mbuf *m)
+{
+ uint64_t eip_typ = IAVF_TX_CTX_DESC_EIPT_NONE;
+ uint64_t eip_len = 0;
+ uint64_t eip_noinc = 0;
+ /* Default - IP_ID is increment in each segment of LSO */
+
+ switch (m->ol_flags & (RTE_MBUF_F_TX_OUTER_IPV4 |
+ RTE_MBUF_F_TX_OUTER_IPV6 |
+ RTE_MBUF_F_TX_OUTER_IP_CKSUM)) {
+ case RTE_MBUF_F_TX_OUTER_IPV4:
+ eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_NO_CHECKSUM_OFFLOAD;
+ eip_len = m->outer_l3_len >> 2;
+ break;
+ case RTE_MBUF_F_TX_OUTER_IPV4 | RTE_MBUF_F_TX_OUTER_IP_CKSUM:
+ eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV4_CHECKSUM_OFFLOAD;
+ eip_len = m->outer_l3_len >> 2;
+ break;
+ case RTE_MBUF_F_TX_OUTER_IPV6:
+ eip_typ = IAVF_TX_CTX_DESC_EIPT_IPV6;
+ eip_len = m->outer_l3_len >> 2;
+ break;
+ }
+
+ /* L4TUNT: L4 Tunneling Type */
+ switch (m->ol_flags & RTE_MBUF_F_TX_TUNNEL_MASK) {
+ case RTE_MBUF_F_TX_TUNNEL_IPIP:
+ /* for non UDP / GRE tunneling, set to 00b */
+ break;
+ case RTE_MBUF_F_TX_TUNNEL_VXLAN:
+ case RTE_MBUF_F_TX_TUNNEL_VXLAN_GPE:
+ case RTE_MBUF_F_TX_TUNNEL_GTP:
+ case RTE_MBUF_F_TX_TUNNEL_GENEVE:
+ eip_typ |= IAVF_TXD_CTX_UDP_TUNNELING;
+ break;
+ case RTE_MBUF_F_TX_TUNNEL_GRE:
+ eip_typ |= IAVF_TXD_CTX_GRE_TUNNELING;
+ break;
+ default:
+ PMD_TX_LOG(ERR, "Tunnel type not supported");
+ return;
+ }
+
+ /* L4TUNLEN: L4 Tunneling Length, in Words
+ *
+ * We depend on app to set rte_mbuf.l2_len correctly.
+ * For IP in GRE it should be set to the length of the GRE
+ * header;
+ * For MAC in GRE or MAC in UDP it should be set to the length
+ * of the GRE or UDP headers plus the inner MAC up to including
+ * its last Ethertype.
+ * If MPLS labels exists, it should include them as well.
+ */
+ eip_typ |= (m->l2_len >> 1) << IAVF_TXD_CTX_QW0_NATLEN_SHIFT;
+
+ /**
+ * Calculate the tunneling UDP checksum.
+ * Shall be set only if L4TUNT = 01b and EIPT is not zero
+ */
+ if ((eip_typ & (IAVF_TX_CTX_EXT_IP_IPV6 |
+ IAVF_TX_CTX_EXT_IP_IPV4 |
+ IAVF_TX_CTX_EXT_IP_IPV4_NO_CSUM)) &&
+ (eip_typ & IAVF_TXD_CTX_UDP_TUNNELING) &&
+ (m->ol_flags & RTE_MBUF_F_TX_OUTER_UDP_CKSUM))
+ eip_typ |= IAVF_TXD_CTX_QW0_L4T_CS_MASK;
+
+ *qw0 = eip_typ << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPT_SHIFT |
+ eip_len << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIPLEN_SHIFT |
+ eip_noinc << IAVF_TXD_CTX_QW0_TUN_PARAMS_EIP_NOINC_SHIFT;
+}
+
static __rte_always_inline void
ctx_vtx1(volatile struct iavf_tx_desc *txdp, struct rte_mbuf *pkt,
uint64_t flags, bool offload, uint8_t vlan_flag)
@@ -2000,6 +2073,8 @@ ctx_vtx(volatile struct iavf_tx_desc *txdp,
if (offload) {
iavf_txd_enable_offload(pkt[1], &hi_data_qw1, vlan_flag);
iavf_txd_enable_offload(pkt[0], &hi_data_qw0, vlan_flag);
+ iavf_fill_ctx_desc_tunnelling_field(&low_ctx_qw1, pkt[1]);
+ iavf_fill_ctx_desc_tunnelling_field(&low_ctx_qw0, pkt[0]);
}
__m512i desc0_3 =
@@ -2239,13 +2314,6 @@ iavf_xmit_pkts_vec_avx512_ctx_cmn(void *tx_queue, struct rte_mbuf **tx_pkts,
return nb_tx;
}
-uint16_t
-iavf_xmit_pkts_vec_avx512_ctx(void *tx_queue, struct rte_mbuf **tx_pkts,
- uint16_t nb_pkts)
-{
- return iavf_xmit_pkts_vec_avx512_ctx_cmn(tx_queue, tx_pkts, nb_pkts, false);
-}
-
uint16_t
iavf_xmit_pkts_vec_avx512_ctx_offload(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
@@ -258,22 +258,19 @@ iavf_tx_vec_queue_default(struct iavf_tx_queue *txq)
* Tunneling parameters and other fields need be configured in ctx desc
* if the outer checksum offload is enabled.
*/
- if (txq->vlan_flag == IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) {
- txq->use_ctx = 1;
- if (txq->offloads & (IAVF_TX_VECTOR_OFFLOAD |
- IAVF_TX_VECTOR_OFFLOAD_CTX))
- return IAVF_VECTOR_CTX_OFFLOAD_PATH;
- else
- return IAVF_VECTOR_CTX_PATH;
- } else {
+ if (txq->offloads & (IAVF_TX_VECTOR_OFFLOAD | IAVF_TX_VECTOR_OFFLOAD_CTX)) {
if (txq->offloads & IAVF_TX_VECTOR_OFFLOAD_CTX) {
- txq->use_ctx = 1;
- return IAVF_VECTOR_CTX_OFFLOAD_PATH;
- } else if (txq->offloads & IAVF_TX_VECTOR_OFFLOAD) {
- return IAVF_VECTOR_OFFLOAD_PATH;
+ if (txq->vlan_flag == IAVF_TX_FLAGS_VLAN_TAG_LOC_L2TAG2) {
+ txq->use_ctx = 1;
+ return IAVF_VECTOR_CTX_OFFLOAD_PATH;
+ } else {
+ return -1;
+ }
} else {
- return IAVF_VECTOR_PATH;
+ return IAVF_VECTOR_OFFLOAD_PATH;
}
+ } else {
+ return IAVF_VECTOR_PATH;
}
}