@@ -5,6 +5,41 @@
#include "cn20k_rx.h"
#include "cn20k_tx.h"
+static uint16_t
+nix_rx_offload_flags(struct rte_eth_dev *eth_dev)
+{
+ struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+ struct rte_eth_dev_data *data = eth_dev->data;
+ struct rte_eth_conf *conf = &data->dev_conf;
+ struct rte_eth_rxmode *rxmode = &conf->rxmode;
+ uint16_t flags = 0;
+
+ if (rxmode->mq_mode == RTE_ETH_MQ_RX_RSS &&
+ (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH))
+ flags |= NIX_RX_OFFLOAD_RSS_F;
+
+ if (dev->rx_offloads & (RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_UDP_CKSUM))
+ flags |= NIX_RX_OFFLOAD_CHECKSUM_F;
+
+ if (dev->rx_offloads &
+ (RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM))
+ flags |= NIX_RX_OFFLOAD_CHECKSUM_F;
+
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
+ flags |= NIX_RX_MULTI_SEG_F;
+
+ if ((dev->rx_offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP))
+ flags |= NIX_RX_OFFLOAD_TSTAMP_F;
+
+ if (!dev->ptype_disable)
+ flags |= NIX_RX_OFFLOAD_PTYPE_F;
+
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY)
+ flags |= NIX_RX_OFFLOAD_SECURITY_F;
+
+ return flags;
+}
+
static int
cn20k_nix_ptypes_set(struct rte_eth_dev *eth_dev, uint32_t ptype_mask)
{
@@ -18,6 +53,7 @@ cn20k_nix_ptypes_set(struct rte_eth_dev *eth_dev, uint32_t ptype_mask)
dev->ptype_disable = 1;
}
+ cn20k_eth_set_rx_function(eth_dev);
return 0;
}
@@ -187,6 +223,9 @@ cn20k_nix_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t qid, uint16_t nb_
rc = nix_recalc_mtu(eth_dev);
if (rc)
return rc;
+
+ /* Update offload flags */
+ dev->rx_offload_flags = nix_rx_offload_flags(eth_dev);
}
rq = &dev->rqs[qid];
@@ -245,6 +284,8 @@ cn20k_nix_configure(struct rte_eth_dev *eth_dev)
if (rc)
return rc;
+ /* Update offload flags */
+ dev->rx_offload_flags = nix_rx_offload_flags(eth_dev);
/* reset reassembly dynfield/flag offset */
dev->reass_dynfield_off = -1;
dev->reass_dynflag_bit = -1;
@@ -271,6 +312,10 @@ cn20k_nix_timesync_enable(struct rte_eth_dev *eth_dev)
for (i = 0; i < eth_dev->data->nb_tx_queues; i++)
nix_form_default_desc(dev, eth_dev->data->tx_queues[i], i);
+ /* Setting up the rx[tx]_offload_flags due to change
+ * in rx[tx]_offloads.
+ */
+ cn20k_eth_set_rx_function(eth_dev);
return 0;
}
@@ -290,6 +335,10 @@ cn20k_nix_timesync_disable(struct rte_eth_dev *eth_dev)
for (i = 0; i < eth_dev->data->nb_tx_queues; i++)
nix_form_default_desc(dev, eth_dev->data->tx_queues[i], i);
+ /* Setting up the rx[tx]_offload_flags due to change
+ * in rx[tx]_offloads.
+ */
+ cn20k_eth_set_rx_function(eth_dev);
return 0;
}
@@ -325,10 +374,15 @@ cn20k_nix_dev_start(struct rte_eth_dev *eth_dev)
if (rc)
return rc;
+ /* Setting up the rx[tx]_offload_flags due to change
+ * in rx[tx]_offloads.
+ */
+ dev->rx_offload_flags |= nix_rx_offload_flags(eth_dev);
/* Set flags for Rx Inject feature */
if (roc_idev_nix_rx_inject_get(nix->port_id))
dev->rx_offload_flags |= NIX_RX_SEC_REASSEMBLY_F;
+ cn20k_eth_set_rx_function(eth_dev);
return 0;
}
@@ -525,8 +579,11 @@ cn20k_nix_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
return -ENOENT;
}
- if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
+ /* Setup callbacks for secondary process */
+ cn20k_eth_set_rx_function(eth_dev);
return 0;
+ }
return 0;
}
@@ -8,4 +8,7 @@
#include <cnxk_ethdev.h>
#include <cnxk_security.h>
+/* Rx and Tx routines */
+void cn20k_eth_set_rx_function(struct rte_eth_dev *eth_dev);
+
#endif /* __CN20K_ETHDEV_H__ */
@@ -30,4 +30,230 @@
#define NIX_RX_MULTI_SEG_F BIT(14)
#define NIX_RX_SEC_REASSEMBLY_F (NIX_RX_REAS_F | NIX_RX_OFFLOAD_SECURITY_F)
+
+#define RSS_F NIX_RX_OFFLOAD_RSS_F
+#define PTYPE_F NIX_RX_OFFLOAD_PTYPE_F
+#define CKSUM_F NIX_RX_OFFLOAD_CHECKSUM_F
+#define MARK_F NIX_RX_OFFLOAD_MARK_UPDATE_F
+#define TS_F NIX_RX_OFFLOAD_TSTAMP_F
+#define RX_VLAN_F NIX_RX_OFFLOAD_VLAN_STRIP_F
+#define R_SEC_F NIX_RX_OFFLOAD_SECURITY_F
+
+/* [R_SEC_F] [RX_VLAN_F] [TS] [MARK] [CKSUM] [PTYPE] [RSS] */
+#define NIX_RX_FASTPATH_MODES_0_15 \
+ R(no_offload, NIX_RX_OFFLOAD_NONE) \
+ R(rss, RSS_F) \
+ R(ptype, PTYPE_F) \
+ R(ptype_rss, PTYPE_F | RSS_F) \
+ R(cksum, CKSUM_F) \
+ R(cksum_rss, CKSUM_F | RSS_F) \
+ R(cksum_ptype, CKSUM_F | PTYPE_F) \
+ R(cksum_ptype_rss, CKSUM_F | PTYPE_F | RSS_F) \
+ R(mark, MARK_F) \
+ R(mark_rss, MARK_F | RSS_F) \
+ R(mark_ptype, MARK_F | PTYPE_F) \
+ R(mark_ptype_rss, MARK_F | PTYPE_F | RSS_F) \
+ R(mark_cksum, MARK_F | CKSUM_F) \
+ R(mark_cksum_rss, MARK_F | CKSUM_F | RSS_F) \
+ R(mark_cksum_ptype, MARK_F | CKSUM_F | PTYPE_F) \
+ R(mark_cksum_ptype_rss, MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_16_31 \
+ R(ts, TS_F) \
+ R(ts_rss, TS_F | RSS_F) \
+ R(ts_ptype, TS_F | PTYPE_F) \
+ R(ts_ptype_rss, TS_F | PTYPE_F | RSS_F) \
+ R(ts_cksum, TS_F | CKSUM_F) \
+ R(ts_cksum_rss, TS_F | CKSUM_F | RSS_F) \
+ R(ts_cksum_ptype, TS_F | CKSUM_F | PTYPE_F) \
+ R(ts_cksum_ptype_rss, TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(ts_mark, TS_F | MARK_F) \
+ R(ts_mark_rss, TS_F | MARK_F | RSS_F) \
+ R(ts_mark_ptype, TS_F | MARK_F | PTYPE_F) \
+ R(ts_mark_ptype_rss, TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(ts_mark_cksum, TS_F | MARK_F | CKSUM_F) \
+ R(ts_mark_cksum_rss, TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(ts_mark_cksum_ptype, TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(ts_mark_cksum_ptype_rss, TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_32_47 \
+ R(vlan, RX_VLAN_F) \
+ R(vlan_rss, RX_VLAN_F | RSS_F) \
+ R(vlan_ptype, RX_VLAN_F | PTYPE_F) \
+ R(vlan_ptype_rss, RX_VLAN_F | PTYPE_F | RSS_F) \
+ R(vlan_cksum, RX_VLAN_F | CKSUM_F) \
+ R(vlan_cksum_rss, RX_VLAN_F | CKSUM_F | RSS_F) \
+ R(vlan_cksum_ptype, RX_VLAN_F | CKSUM_F | PTYPE_F) \
+ R(vlan_cksum_ptype_rss, RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(vlan_mark, RX_VLAN_F | MARK_F) \
+ R(vlan_mark_rss, RX_VLAN_F | MARK_F | RSS_F) \
+ R(vlan_mark_ptype, RX_VLAN_F | MARK_F | PTYPE_F) \
+ R(vlan_mark_ptype_rss, RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \
+ R(vlan_mark_cksum, RX_VLAN_F | MARK_F | CKSUM_F) \
+ R(vlan_mark_cksum_rss, RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \
+ R(vlan_mark_cksum_ptype, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(vlan_mark_cksum_ptype_rss, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_48_63 \
+ R(vlan_ts, RX_VLAN_F | TS_F) \
+ R(vlan_ts_rss, RX_VLAN_F | TS_F | RSS_F) \
+ R(vlan_ts_ptype, RX_VLAN_F | TS_F | PTYPE_F) \
+ R(vlan_ts_ptype_rss, RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
+ R(vlan_ts_cksum, RX_VLAN_F | TS_F | CKSUM_F) \
+ R(vlan_ts_cksum_rss, RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
+ R(vlan_ts_cksum_ptype, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \
+ R(vlan_ts_cksum_ptype_rss, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(vlan_ts_mark, RX_VLAN_F | TS_F | MARK_F) \
+ R(vlan_ts_mark_rss, RX_VLAN_F | TS_F | MARK_F | RSS_F) \
+ R(vlan_ts_mark_ptype, RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \
+ R(vlan_ts_mark_ptype_rss, RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(vlan_ts_mark_cksum, RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \
+ R(vlan_ts_mark_cksum_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(vlan_ts_mark_cksum_ptype, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(vlan_ts_mark_cksum_ptype_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_64_79 \
+ R(sec, R_SEC_F) \
+ R(sec_rss, R_SEC_F | RSS_F) \
+ R(sec_ptype, R_SEC_F | PTYPE_F) \
+ R(sec_ptype_rss, R_SEC_F | PTYPE_F | RSS_F) \
+ R(sec_cksum, R_SEC_F | CKSUM_F) \
+ R(sec_cksum_rss, R_SEC_F | CKSUM_F | RSS_F) \
+ R(sec_cksum_ptype, R_SEC_F | CKSUM_F | PTYPE_F) \
+ R(sec_cksum_ptype_rss, R_SEC_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_mark, R_SEC_F | MARK_F) \
+ R(sec_mark_rss, R_SEC_F | MARK_F | RSS_F) \
+ R(sec_mark_ptype, R_SEC_F | MARK_F | PTYPE_F) \
+ R(sec_mark_ptype_rss, R_SEC_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_mark_cksum, R_SEC_F | MARK_F | CKSUM_F) \
+ R(sec_mark_cksum_rss, R_SEC_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_mark_cksum_ptype, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_mark_cksum_ptype_rss, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_80_95 \
+ R(sec_ts, R_SEC_F | TS_F) \
+ R(sec_ts_rss, R_SEC_F | TS_F | RSS_F) \
+ R(sec_ts_ptype, R_SEC_F | TS_F | PTYPE_F) \
+ R(sec_ts_ptype_rss, R_SEC_F | TS_F | PTYPE_F | RSS_F) \
+ R(sec_ts_cksum, R_SEC_F | TS_F | CKSUM_F) \
+ R(sec_ts_cksum_rss, R_SEC_F | TS_F | CKSUM_F | RSS_F) \
+ R(sec_ts_cksum_ptype, R_SEC_F | TS_F | CKSUM_F | PTYPE_F) \
+ R(sec_ts_cksum_ptype_rss, R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_ts_mark, R_SEC_F | TS_F | MARK_F) \
+ R(sec_ts_mark_rss, R_SEC_F | TS_F | MARK_F | RSS_F) \
+ R(sec_ts_mark_ptype, R_SEC_F | TS_F | MARK_F | PTYPE_F) \
+ R(sec_ts_mark_ptype_rss, R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_ts_mark_cksum, R_SEC_F | TS_F | MARK_F | CKSUM_F) \
+ R(sec_ts_mark_cksum_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_ts_mark_cksum_ptype, R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_ts_mark_cksum_ptype_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_96_111 \
+ R(sec_vlan, R_SEC_F | RX_VLAN_F) \
+ R(sec_vlan_rss, R_SEC_F | RX_VLAN_F | RSS_F) \
+ R(sec_vlan_ptype, R_SEC_F | RX_VLAN_F | PTYPE_F) \
+ R(sec_vlan_ptype_rss, R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_cksum, R_SEC_F | RX_VLAN_F | CKSUM_F) \
+ R(sec_vlan_cksum_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_cksum_ptype, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_cksum_ptype_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_mark, R_SEC_F | RX_VLAN_F | MARK_F) \
+ R(sec_vlan_mark_rss, R_SEC_F | RX_VLAN_F | MARK_F | RSS_F) \
+ R(sec_vlan_mark_ptype, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F) \
+ R(sec_vlan_mark_ptype_rss, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_mark_cksum, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F) \
+ R(sec_vlan_mark_cksum_rss, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_mark_cksum_ptype, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_mark_cksum_ptype_rss, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_112_127 \
+ R(sec_vlan_ts, R_SEC_F | RX_VLAN_F | TS_F) \
+ R(sec_vlan_ts_rss, R_SEC_F | RX_VLAN_F | TS_F | RSS_F) \
+ R(sec_vlan_ts_ptype, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F) \
+ R(sec_vlan_ts_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_ts_cksum, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F) \
+ R(sec_vlan_ts_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_ts_cksum_ptype, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_ts_cksum_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_ts_mark, R_SEC_F | RX_VLAN_F | TS_F | MARK_F) \
+ R(sec_vlan_ts_mark_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F) \
+ R(sec_vlan_ts_mark_ptype, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \
+ R(sec_vlan_ts_mark_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_ts_mark_cksum, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \
+ R(sec_vlan_ts_mark_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_ts_mark_cksum_ptype, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_ts_mark_cksum_ptype_rss, \
+ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES \
+ NIX_RX_FASTPATH_MODES_0_15 \
+ NIX_RX_FASTPATH_MODES_16_31 \
+ NIX_RX_FASTPATH_MODES_32_47 \
+ NIX_RX_FASTPATH_MODES_48_63 \
+ NIX_RX_FASTPATH_MODES_64_79 \
+ NIX_RX_FASTPATH_MODES_80_95 \
+ NIX_RX_FASTPATH_MODES_96_111 \
+ NIX_RX_FASTPATH_MODES_112_127
+
+#define R(name, flags) \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_mseg_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_vec_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_vec_mseg_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_reas_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_reas_mseg_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_reas_vec_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
+ uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_reas_vec_mseg_##name( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);
+
+NIX_RX_FASTPATH_MODES
+#undef R
+
+#define NIX_RX_RECV(fn, flags) \
+ uint16_t __rte_noinline __rte_hot fn(void *rx_queue, struct rte_mbuf **rx_pkts, \
+ uint16_t pkts) \
+ { \
+ RTE_SET_USED(rx_queue); \
+ RTE_SET_USED(rx_pkts); \
+ RTE_SET_USED(pkts); \
+ return 0; \
+ }
+
+#define NIX_RX_RECV_MSEG(fn, flags) NIX_RX_RECV(fn, flags | NIX_RX_MULTI_SEG_F)
+
+#define NIX_RX_RECV_VEC(fn, flags) \
+ uint16_t __rte_noinline __rte_hot fn(void *rx_queue, struct rte_mbuf **rx_pkts, \
+ uint16_t pkts) \
+ { \
+ RTE_SET_USED(rx_queue); \
+ RTE_SET_USED(rx_pkts); \
+ RTE_SET_USED(pkts); \
+ return 0; \
+ }
+
+#define NIX_RX_RECV_VEC_MSEG(fn, flags) NIX_RX_RECV_VEC(fn, flags | NIX_RX_MULTI_SEG_F)
+
+uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_all_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t pkts);
+
+uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_vec_all_offload(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t pkts);
+
+uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_all_offload_tst(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t pkts);
+
+uint16_t __rte_noinline __rte_hot cn20k_nix_recv_pkts_vec_all_offload_tst(void *rx_queue,
+ struct rte_mbuf **rx_pkts,
+ uint16_t pkts);
+
#endif /* __CN20K_RX_H__ */
new file mode 100644
@@ -0,0 +1,162 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_ethdev.h"
+#include "cn20k_rx.h"
+
+static __rte_used void
+pick_rx_func(struct rte_eth_dev *eth_dev, const eth_rx_burst_t rx_burst[NIX_RX_OFFLOAD_MAX])
+{
+ struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+
+ /* [VLAN] [TSP] [MARK] [CKSUM] [PTYPE] [RSS] */
+ eth_dev->rx_pkt_burst = rx_burst[dev->rx_offload_flags & (NIX_RX_OFFLOAD_MAX - 1)];
+
+ if (eth_dev->data->dev_started)
+ rte_eth_fp_ops[eth_dev->data->port_id].rx_pkt_burst = eth_dev->rx_pkt_burst;
+
+ rte_atomic_thread_fence(rte_memory_order_release);
+}
+
+static uint16_t __rte_noinline __rte_hot __rte_unused
+cn20k_nix_flush_rx(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)
+{
+ RTE_SET_USED(rx_queue);
+ RTE_SET_USED(rx_pkts);
+ RTE_SET_USED(pkts);
+ return 0;
+}
+
+#if defined(RTE_ARCH_ARM64)
+static void
+cn20k_eth_set_rx_tmplt_func(struct rte_eth_dev *eth_dev)
+{
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+ struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+
+ const eth_rx_burst_t nix_eth_rx_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_burst_mseg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_mseg_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_burst_reas[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_reas_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_burst_mseg_reas[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_reas_mseg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_vec_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_vec_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_vec_burst_mseg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_vec_mseg_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_vec_burst_reas[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_reas_vec_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_vec_burst_mseg_reas[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn20k_nix_recv_pkts_reas_vec_mseg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ /* Copy multi seg version with security for tear down sequence */
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ dev->rx_pkt_burst_no_offload = cn20k_nix_flush_rx;
+
+ if (dev->scalar_ena) {
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER) {
+ if (dev->rx_offload_flags & NIX_RX_REAS_F)
+ return pick_rx_func(eth_dev, nix_eth_rx_burst_mseg_reas);
+ else
+ return pick_rx_func(eth_dev, nix_eth_rx_burst_mseg);
+ }
+ if (dev->rx_offload_flags & NIX_RX_REAS_F)
+ return pick_rx_func(eth_dev, nix_eth_rx_burst_reas);
+ else
+ return pick_rx_func(eth_dev, nix_eth_rx_burst);
+ }
+
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER) {
+ if (dev->rx_offload_flags & NIX_RX_REAS_F)
+ return pick_rx_func(eth_dev, nix_eth_rx_vec_burst_mseg_reas);
+ else
+ return pick_rx_func(eth_dev, nix_eth_rx_vec_burst_mseg);
+ }
+
+ if (dev->rx_offload_flags & NIX_RX_REAS_F)
+ return pick_rx_func(eth_dev, nix_eth_rx_vec_burst_reas);
+ else
+ return pick_rx_func(eth_dev, nix_eth_rx_vec_burst);
+#else
+ RTE_SET_USED(eth_dev);
+#endif
+}
+
+static void
+cn20k_eth_set_rx_blk_func(struct rte_eth_dev *eth_dev)
+{
+#if defined(CNXK_DIS_TMPLT_FUNC)
+ struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+
+ /* Copy multi seg version with security for tear down sequence */
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ dev->rx_pkt_burst_no_offload = cn20k_nix_flush_rx;
+
+ if (dev->scalar_ena) {
+ eth_dev->rx_pkt_burst = cn20k_nix_recv_pkts_all_offload;
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)
+ eth_dev->rx_pkt_burst = cn20k_nix_recv_pkts_all_offload_tst;
+ } else {
+ eth_dev->rx_pkt_burst = cn20k_nix_recv_pkts_vec_all_offload;
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)
+ eth_dev->rx_pkt_burst = cn20k_nix_recv_pkts_vec_all_offload_tst;
+ }
+
+ if (eth_dev->data->dev_started)
+ rte_eth_fp_ops[eth_dev->data->port_id].rx_pkt_burst = eth_dev->rx_pkt_burst;
+#else
+ RTE_SET_USED(eth_dev);
+#endif
+}
+#endif
+
+void
+cn20k_eth_set_rx_function(struct rte_eth_dev *eth_dev)
+{
+#if defined(RTE_ARCH_ARM64)
+ cn20k_eth_set_rx_blk_func(eth_dev);
+ cn20k_eth_set_rx_tmplt_func(eth_dev);
+
+ rte_atomic_thread_fence(rte_memory_order_release);
+#else
+ RTE_SET_USED(eth_dev);
+#endif
+}
@@ -236,7 +236,51 @@ if soc_type == 'cn20k' or soc_type == 'all'
# CN20K
sources += files(
'cn20k_ethdev.c',
+ 'cn20k_rx_select.c',
)
+
+if host_machine.cpu_family().startswith('aarch') and not disable_template
+sources += files(
+ 'rx/cn20k/rx_0_15.c',
+ 'rx/cn20k/rx_16_31.c',
+ 'rx/cn20k/rx_32_47.c',
+ 'rx/cn20k/rx_48_63.c',
+ 'rx/cn20k/rx_64_79.c',
+ 'rx/cn20k/rx_80_95.c',
+ 'rx/cn20k/rx_96_111.c',
+ 'rx/cn20k/rx_112_127.c',
+ 'rx/cn20k/rx_0_15_mseg.c',
+ 'rx/cn20k/rx_16_31_mseg.c',
+ 'rx/cn20k/rx_32_47_mseg.c',
+ 'rx/cn20k/rx_48_63_mseg.c',
+ 'rx/cn20k/rx_64_79_mseg.c',
+ 'rx/cn20k/rx_80_95_mseg.c',
+ 'rx/cn20k/rx_96_111_mseg.c',
+ 'rx/cn20k/rx_112_127_mseg.c',
+ 'rx/cn20k/rx_0_15_vec.c',
+ 'rx/cn20k/rx_16_31_vec.c',
+ 'rx/cn20k/rx_32_47_vec.c',
+ 'rx/cn20k/rx_48_63_vec.c',
+ 'rx/cn20k/rx_64_79_vec.c',
+ 'rx/cn20k/rx_80_95_vec.c',
+ 'rx/cn20k/rx_96_111_vec.c',
+ 'rx/cn20k/rx_112_127_vec.c',
+ 'rx/cn20k/rx_0_15_vec_mseg.c',
+ 'rx/cn20k/rx_16_31_vec_mseg.c',
+ 'rx/cn20k/rx_32_47_vec_mseg.c',
+ 'rx/cn20k/rx_48_63_vec_mseg.c',
+ 'rx/cn20k/rx_64_79_vec_mseg.c',
+ 'rx/cn20k/rx_80_95_vec_mseg.c',
+ 'rx/cn20k/rx_96_111_vec_mseg.c',
+ 'rx/cn20k/rx_112_127_vec_mseg.c',
+ 'rx/cn20k/rx_all_offload.c',
+)
+
+else
+sources += files(
+ 'rx/cn20k/rx_all_offload.c',
+)
+endif
endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_0_15
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_0_15
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_0_15
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_0_15
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_112_127
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_112_127
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_112_127
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_112_127
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_16_31
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_16_31
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_16_31
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_16_31
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_32_47
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_32_47
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_32_47
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_32_47
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_48_63
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_48_63
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_48_63
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_48_63
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_64_79
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_64_79
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_64_79
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_64_79
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_80_95
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_80_95
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_80_95
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_80_95
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_##name, flags) \
+ NIX_RX_RECV(cn20k_nix_recv_pkts_reas_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_96_111
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_mseg_##name, flags) \
+ NIX_RX_RECV_MSEG(cn20k_nix_recv_pkts_reas_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_96_111
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_vec_##name, flags) \
+ NIX_RX_RECV_VEC(cn20k_nix_recv_pkts_reas_vec_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_96_111
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if !defined(CNXK_DIS_TMPLT_FUNC)
+
+#define R(name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_vec_mseg_##name, flags) \
+ NIX_RX_RECV_VEC_MSEG(cn20k_nix_recv_pkts_reas_vec_mseg_##name, flags | NIX_RX_REAS_F)
+
+NIX_RX_FASTPATH_MODES_96_111
+#undef R
+
+#endif
new file mode 100644
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2024 Marvell.
+ */
+
+#include "cn20k_rx.h"
+
+#ifdef _ROC_API_H_
+#error "roc_api.h is included"
+#endif
+
+#if defined(CNXK_DIS_TMPLT_FUNC)
+
+uint16_t __rte_noinline __rte_hot
+cn20k_nix_recv_pkts_all_offload(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)
+{
+ return cn20k_nix_recv_pkts(rx_queue, rx_pkts, pkts,
+ NIX_RX_OFFLOAD_RSS_F | NIX_RX_OFFLOAD_PTYPE_F |
+ NIX_RX_OFFLOAD_CHECKSUM_F |
+ NIX_RX_OFFLOAD_MARK_UPDATE_F | NIX_RX_OFFLOAD_TSTAMP_F |
+ NIX_RX_OFFLOAD_VLAN_STRIP_F | NIX_RX_OFFLOAD_SECURITY_F |
+ NIX_RX_MULTI_SEG_F | NIX_RX_REAS_F);
+}
+
+uint16_t __rte_noinline __rte_hot
+cn20k_nix_recv_pkts_vec_all_offload(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)
+{
+ return cn20k_nix_recv_pkts_vector(
+ rx_queue, rx_pkts, pkts,
+ NIX_RX_OFFLOAD_RSS_F | NIX_RX_OFFLOAD_PTYPE_F | NIX_RX_OFFLOAD_CHECKSUM_F |
+ NIX_RX_OFFLOAD_MARK_UPDATE_F | NIX_RX_OFFLOAD_TSTAMP_F |
+ NIX_RX_OFFLOAD_VLAN_STRIP_F | NIX_RX_OFFLOAD_SECURITY_F |
+ NIX_RX_MULTI_SEG_F | NIX_RX_REAS_F,
+ NULL, NULL, 0, 0);
+}
+
+uint16_t __rte_noinline __rte_hot
+cn20k_nix_recv_pkts_all_offload_tst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)
+{
+ return cn20k_nix_recv_pkts(
+ rx_queue, rx_pkts, pkts,
+ NIX_RX_OFFLOAD_RSS_F | NIX_RX_OFFLOAD_PTYPE_F | NIX_RX_OFFLOAD_CHECKSUM_F |
+ NIX_RX_OFFLOAD_MARK_UPDATE_F | NIX_RX_OFFLOAD_VLAN_STRIP_F |
+ NIX_RX_OFFLOAD_SECURITY_F | NIX_RX_MULTI_SEG_F | NIX_RX_REAS_F);
+}
+
+uint16_t __rte_noinline __rte_hot
+cn20k_nix_recv_pkts_vec_all_offload_tst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts)
+{
+ return cn20k_nix_recv_pkts_vector(
+ rx_queue, rx_pkts, pkts,
+ NIX_RX_OFFLOAD_RSS_F | NIX_RX_OFFLOAD_PTYPE_F | NIX_RX_OFFLOAD_CHECKSUM_F |
+ NIX_RX_OFFLOAD_MARK_UPDATE_F | NIX_RX_OFFLOAD_VLAN_STRIP_F |
+ NIX_RX_OFFLOAD_SECURITY_F | NIX_RX_MULTI_SEG_F | NIX_RX_REAS_F,
+ NULL, NULL, 0, 0);
+}
+
+#endif