@@ -10,13 +10,7 @@
#define CN9K_DUAL_WS_PAIR_ID(x, id) (((x)*CN9K_DUAL_WS_NB_WS) + id)
#define CN9K_SET_EVDEV_DEQ_OP(dev, deq_op, deq_ops) \
- (deq_op = deq_ops[!!(dev->rx_offloads & NIX_RX_OFFLOAD_SECURITY_F)] \
- [!!(dev->rx_offloads & NIX_RX_OFFLOAD_VLAN_STRIP_F)] \
- [!!(dev->rx_offloads & NIX_RX_OFFLOAD_TSTAMP_F)] \
- [!!(dev->rx_offloads & NIX_RX_OFFLOAD_MARK_UPDATE_F)] \
- [!!(dev->rx_offloads & NIX_RX_OFFLOAD_CHECKSUM_F)] \
- [!!(dev->rx_offloads & NIX_RX_OFFLOAD_PTYPE_F)] \
- [!!(dev->rx_offloads & NIX_RX_OFFLOAD_RSS_F)])
+ deq_op = deq_ops[dev->rx_offloads & (NIX_RX_OFFLOAD_MAX - 1)]
#define CN9K_SET_EVDEV_ENQ_OP(dev, enq_op, enq_ops) \
(enq_op = enq_ops[!!(dev->tx_offloads & NIX_TX_OFFLOAD_SECURITY_F)] \
@@ -316,188 +310,214 @@ cn9k_sso_fp_fns_set(struct rte_eventdev *event_dev)
{
struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
/* Single WS modes */
- const event_dequeue_t sso_hws_deq[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_##name,
+ const event_dequeue_t sso_hws_deq[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_burst_t sso_hws_deq_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_burst_##name,
+ const event_dequeue_burst_t sso_hws_deq_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_t sso_hws_deq_tmo[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_tmo_##name,
+ const event_dequeue_t sso_hws_deq_tmo[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_burst_t
- sso_hws_deq_tmo_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_tmo_burst_##name,
- NIX_RX_FASTPATH_MODES
+ const event_dequeue_burst_t sso_hws_deq_tmo_burst[NIX_RX_OFFLOAD_MAX] =
+ {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_burst_##name,
+ NIX_RX_FASTPATH_MODES
#undef R
- };
+ };
- const event_dequeue_t sso_hws_deq_ca[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_ca_##name,
+ const event_dequeue_t sso_hws_deq_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_ca_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_burst_t
- sso_hws_deq_ca_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_ca_burst_##name,
+ const event_dequeue_burst_t sso_hws_deq_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_ca_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_t sso_hws_deq_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_seg_##name,
+ const event_dequeue_t sso_hws_deq_tmo_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_ca_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_deq_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_seg_burst_##name,
+ sso_hws_deq_tmo_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_ca_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_seg_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_t sso_hws_deq_tmo_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_tmo_seg_##name,
+ const event_dequeue_burst_t sso_hws_deq_seg_burst[NIX_RX_OFFLOAD_MAX] =
+ {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_deq_tmo_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_seg_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_deq_tmo_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_tmo_seg_burst_##name,
+ sso_hws_deq_tmo_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_seg_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
- };
+ };
- const event_dequeue_t sso_hws_deq_ca_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_ca_seg_##name,
+ const event_dequeue_t sso_hws_deq_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_ca_seg_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_deq_ca_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_deq_ca_seg_burst_##name,
+ sso_hws_deq_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_ca_seg_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
- };
+ };
- /* Dual WS modes */
- const event_dequeue_t sso_hws_dual_deq[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_##name,
+ const event_dequeue_t sso_hws_deq_tmo_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_ca_seg_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_dual_deq_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_burst_##name,
+ sso_hws_deq_tmo_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_deq_tmo_ca_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ /* Dual WS modes */
+ const event_dequeue_t sso_hws_dual_deq[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_t sso_hws_dual_deq_tmo[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_tmo_##name,
+ const event_dequeue_burst_t sso_hws_dual_deq_burst[NIX_RX_OFFLOAD_MAX] =
+ {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_tmo[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_dual_deq_tmo_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_tmo_burst_##name,
+ sso_hws_dual_deq_tmo_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
- };
+ };
- const event_dequeue_t sso_hws_dual_deq_ca[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_ca_##name,
+ const event_dequeue_t sso_hws_dual_deq_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_ca_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_dual_deq_ca_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_ca_burst_##name,
+ sso_hws_dual_deq_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_ca_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
- };
+ };
- const event_dequeue_t sso_hws_dual_deq_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_seg_##name,
+ const event_dequeue_t sso_hws_dual_deq_tmo_ca[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_ca_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_dual_deq_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_seg_burst_##name,
+ sso_hws_dual_deq_tmo_ca_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_ca_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_t sso_hws_dual_deq_tmo_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_tmo_seg_##name,
+ const event_dequeue_t sso_hws_dual_deq_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_seg_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_dual_deq_tmo_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = \
- cn9k_sso_hws_dual_deq_tmo_seg_burst_##name,
+ sso_hws_dual_deq_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_seg_burst_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
- const event_dequeue_t sso_hws_dual_deq_ca_seg[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = cn9k_sso_hws_dual_deq_ca_seg_##name,
+ const event_dequeue_t sso_hws_dual_deq_tmo_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_seg_##name,
NIX_RX_FASTPATH_MODES
#undef R
};
const event_dequeue_burst_t
- sso_hws_dual_deq_ca_seg_burst[2][2][2][2][2][2][2] = {
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- [f6][f5][f4][f3][f2][f1][f0] = \
- cn9k_sso_hws_dual_deq_ca_seg_burst_##name,
+ sso_hws_dual_deq_tmo_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_seg_burst_##name,
NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_ca_seg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_ca_seg_##name,
+ NIX_RX_FASTPATH_MODES
#undef R
};
+ const event_dequeue_burst_t
+ sso_hws_dual_deq_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_ca_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_t sso_hws_dual_deq_tmo_ca_seg[NIX_RX_OFFLOAD_MAX] =
+ {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_ca_seg_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const event_dequeue_burst_t
+ sso_hws_dual_deq_tmo_ca_seg_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_sso_hws_dual_deq_tmo_ca_seg_burst_##name,
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
/* Tx modes */
const event_tx_adapter_enqueue_t
sso_hws_tx_adptr_enq[2][2][2][2][2][2][2] = {
@@ -552,6 +572,13 @@ cn9k_sso_fp_fns_set(struct rte_eventdev *event_dev)
CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
sso_hws_deq_ca_seg_burst);
}
+
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_tmo_ca_seg);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_tmo_ca_seg_burst);
+ }
} else {
CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue, sso_hws_deq);
CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
@@ -568,6 +595,13 @@ cn9k_sso_fp_fns_set(struct rte_eventdev *event_dev)
CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
sso_hws_deq_ca_burst);
}
+
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_deq_tmo_ca);
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue_burst,
+ sso_hws_deq_tmo_ca_burst);
+ }
}
event_dev->ca_enqueue = cn9k_sso_hws_ca_enq;
@@ -605,6 +639,14 @@ cn9k_sso_fp_fns_set(struct rte_eventdev *event_dev)
dev, event_dev->dequeue_burst,
sso_hws_dual_deq_ca_seg_burst);
}
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue,
+ sso_hws_dual_deq_tmo_ca_seg);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_tmo_ca_seg_burst);
+ }
} else {
CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
sso_hws_dual_deq);
@@ -624,6 +666,13 @@ cn9k_sso_fp_fns_set(struct rte_eventdev *event_dev)
dev, event_dev->dequeue_burst,
sso_hws_dual_deq_ca_burst);
}
+ if (dev->is_ca_internal_port && dev->is_timeout_deq) {
+ CN9K_SET_EVDEV_DEQ_OP(dev, event_dev->dequeue,
+ sso_hws_dual_deq_tmo_ca);
+ CN9K_SET_EVDEV_DEQ_OP(
+ dev, event_dev->dequeue_burst,
+ sso_hws_dual_deq_tmo_ca_burst);
+ }
}
if (dev->tx_offloads & NIX_TX_MULTI_SEG_F)
@@ -385,7 +385,7 @@ uint16_t __rte_hot cn9k_sso_hws_ca_enq(void *port, struct rte_event ev[],
uint16_t __rte_hot cn9k_sso_hws_dual_ca_enq(void *port, struct rte_event ev[],
uint16_t nb_events);
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
+#define R(name, flags) \
uint16_t __rte_hot cn9k_sso_hws_deq_##name( \
void *port, struct rte_event *ev, uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_deq_burst_##name( \
@@ -401,6 +401,11 @@ uint16_t __rte_hot cn9k_sso_hws_dual_ca_enq(void *port, struct rte_event ev[],
uint16_t __rte_hot cn9k_sso_hws_deq_ca_burst_##name( \
void *port, struct rte_event ev[], uint16_t nb_events, \
uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_deq_tmo_ca_##name( \
+ void *port, struct rte_event *ev, uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_deq_tmo_ca_burst_##name( \
+ void *port, struct rte_event ev[], uint16_t nb_events, \
+ uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_deq_seg_##name( \
void *port, struct rte_event *ev, uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_deq_seg_burst_##name( \
@@ -414,13 +419,61 @@ uint16_t __rte_hot cn9k_sso_hws_dual_ca_enq(void *port, struct rte_event ev[],
uint16_t __rte_hot cn9k_sso_hws_deq_ca_seg_##name( \
void *port, struct rte_event *ev, uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_deq_ca_seg_burst_##name( \
+ void *port, struct rte_event ev[], uint16_t nb_events, \
+ uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_deq_tmo_ca_seg_##name( \
+ void *port, struct rte_event *ev, uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_deq_tmo_ca_seg_burst_##name( \
void *port, struct rte_event ev[], uint16_t nb_events, \
uint64_t timeout_ticks);
NIX_RX_FASTPATH_MODES
#undef R
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
+#define SSO_DEQ(fn, flags) \
+ uint16_t __rte_hot fn(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+ { \
+ struct cn9k_sso_hws *ws = port; \
+ \
+ RTE_SET_USED(timeout_ticks); \
+ if (ws->swtag_req) { \
+ ws->swtag_req = 0; \
+ cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
+ return 1; \
+ } \
+ return cn9k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \
+ }
+
+#define SSO_DEQ_SEG(fn, flags) SSO_DEQ(fn, flags | NIX_RX_MULTI_SEG_F)
+#define SSO_DEQ_CA(fn, flags) SSO_DEQ(fn, flags | CPT_RX_WQE_F)
+#define SSO_DEQ_CA_SEG(fn, flags) SSO_DEQ_SEG(fn, flags | CPT_RX_WQE_F)
+
+#define SSO_DEQ_TMO(fn, flags) \
+ uint16_t __rte_hot fn(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+ { \
+ struct cn9k_sso_hws *ws = port; \
+ uint16_t ret = 1; \
+ uint64_t iter; \
+ \
+ if (ws->swtag_req) { \
+ ws->swtag_req = 0; \
+ cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
+ return ret; \
+ } \
+ ret = cn9k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \
+ for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \
+ ret = cn9k_sso_hws_get_work(ws, ev, flags, \
+ ws->lookup_mem); \
+ return ret; \
+ }
+
+#define SSO_DEQ_TMO_SEG(fn, flags) SSO_DEQ_TMO(fn, flags | NIX_RX_MULTI_SEG_F)
+#define SSO_DEQ_TMO_CA(fn, flags) SSO_DEQ_TMO(fn, flags | CPT_RX_WQE_F)
+#define SSO_DEQ_TMO_CA_SEG(fn, flags) SSO_DEQ_TMO_SEG(fn, flags | CPT_RX_WQE_F)
+
+#define R(name, flags) \
uint16_t __rte_hot cn9k_sso_hws_dual_deq_##name( \
void *port, struct rte_event *ev, uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_dual_deq_burst_##name( \
@@ -436,6 +489,11 @@ NIX_RX_FASTPATH_MODES
uint16_t __rte_hot cn9k_sso_hws_dual_deq_ca_burst_##name( \
void *port, struct rte_event ev[], uint16_t nb_events, \
uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_ca_##name( \
+ void *port, struct rte_event *ev, uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_ca_burst_##name( \
+ void *port, struct rte_event ev[], uint16_t nb_events, \
+ uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_dual_deq_seg_##name( \
void *port, struct rte_event *ev, uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_dual_deq_seg_burst_##name( \
@@ -449,12 +507,93 @@ NIX_RX_FASTPATH_MODES
uint16_t __rte_hot cn9k_sso_hws_dual_deq_ca_seg_##name( \
void *port, struct rte_event *ev, uint64_t timeout_ticks); \
uint16_t __rte_hot cn9k_sso_hws_dual_deq_ca_seg_burst_##name( \
+ void *port, struct rte_event ev[], uint16_t nb_events, \
+ uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_ca_seg_##name( \
+ void *port, struct rte_event *ev, uint64_t timeout_ticks); \
+ uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_ca_seg_burst_##name( \
void *port, struct rte_event ev[], uint16_t nb_events, \
uint64_t timeout_ticks);
NIX_RX_FASTPATH_MODES
#undef R
+#define SSO_DUAL_DEQ(fn, flags) \
+ uint16_t __rte_hot fn(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+ { \
+ struct cn9k_sso_hws_dual *dws = port; \
+ uint16_t gw; \
+ \
+ RTE_SET_USED(timeout_ticks); \
+ if (dws->swtag_req) { \
+ dws->swtag_req = 0; \
+ cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
+ SSOW_LF_GWS_TAG); \
+ return 1; \
+ } \
+ gw = cn9k_sso_hws_dual_get_work( \
+ dws->base[dws->vws], dws->base[!dws->vws], ev, flags, \
+ dws->lookup_mem, dws->tstamp); \
+ dws->vws = !dws->vws; \
+ return gw; \
+ }
+
+#define SSO_DUAL_DEQ_SEG(fn, flags) SSO_DUAL_DEQ(fn, flags | NIX_RX_MULTI_SEG_F)
+#define SSO_DUAL_DEQ_CA(fn, flags) SSO_DUAL_DEQ(fn, flags | CPT_RX_WQE_F)
+#define SSO_DUAL_DEQ_CA_SEG(fn, flags) \
+ SSO_DUAL_DEQ_SEG(fn, flags | CPT_RX_WQE_F)
+
+#define SSO_DUAL_DEQ_TMO(fn, flags) \
+ uint16_t __rte_hot fn(void *port, struct rte_event *ev, \
+ uint64_t timeout_ticks) \
+ { \
+ struct cn9k_sso_hws_dual *dws = port; \
+ uint16_t ret = 1; \
+ uint64_t iter; \
+ \
+ if (dws->swtag_req) { \
+ dws->swtag_req = 0; \
+ cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
+ SSOW_LF_GWS_TAG); \
+ return ret; \
+ } \
+ ret = cn9k_sso_hws_dual_get_work( \
+ dws->base[dws->vws], dws->base[!dws->vws], ev, flags, \
+ dws->lookup_mem, dws->tstamp); \
+ dws->vws = !dws->vws; \
+ for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) { \
+ ret = cn9k_sso_hws_dual_get_work( \
+ dws->base[dws->vws], dws->base[!dws->vws], ev, \
+ flags, dws->lookup_mem, dws->tstamp); \
+ dws->vws = !dws->vws; \
+ } \
+ return ret; \
+ }
+
+#define SSO_DUAL_DEQ_TMO_SEG(fn, flags) \
+ SSO_DUAL_DEQ_TMO(fn, flags | NIX_RX_MULTI_SEG_F)
+#define SSO_DUAL_DEQ_TMO_CA(fn, flags) \
+ SSO_DUAL_DEQ_TMO(fn, flags | CPT_RX_WQE_F)
+#define SSO_DUAL_DEQ_TMO_CA_SEG(fn, flags) \
+ SSO_DUAL_DEQ_TMO_SEG(fn, flags | CPT_RX_WQE_F)
+
+#define SSO_CMN_DEQ_BURST(fnb, fn, flags) \
+ uint16_t __rte_hot fnb(void *port, struct rte_event ev[], \
+ uint16_t nb_events, uint64_t timeout_ticks) \
+ { \
+ RTE_SET_USED(nb_events); \
+ return fn(port, ev, timeout_ticks); \
+ }
+
+#define SSO_CMN_DEQ_SEG_BURST(fnb, fn, flags) \
+ uint16_t __rte_hot fnb(void *port, struct rte_event ev[], \
+ uint16_t nb_events, uint64_t timeout_ticks) \
+ { \
+ RTE_SET_USED(nb_events); \
+ return fn(port, ev, timeout_ticks); \
+ }
+
static __rte_always_inline void
cn9k_sso_txq_fc_wait(const struct cn9k_eth_txq *txq)
{
deleted file mode 100644
@@ -1,44 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_deq_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws *ws = port; \
- \
- RTE_SET_USED(timeout_ticks); \
- \
- if (ws->swtag_req) { \
- ws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- return cn9k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_seg_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws *ws = port; \
- \
- RTE_SET_USED(timeout_ticks); \
- \
- if (ws->swtag_req) { \
- ws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- return cn9k_sso_hws_get_work( \
- ws, ev, flags | NIX_RX_MULTI_SEG_F, ws->lookup_mem); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,29 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_deq_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_deq_##name(port, ev, timeout_ticks); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_seg_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_deq_seg_##name(port, ev, timeout_ticks); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,65 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_deq_ca_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws *ws = port; \
- \
- RTE_SET_USED(timeout_ticks); \
- \
- if (ws->swtag_req) { \
- ws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- return cn9k_sso_hws_get_work(ws, ev, flags | CPT_RX_WQE_F, \
- ws->lookup_mem); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_ca_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_deq_ca_##name(port, ev, timeout_ticks); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_ca_seg_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws *ws = port; \
- \
- RTE_SET_USED(timeout_ticks); \
- \
- if (ws->swtag_req) { \
- ws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- return cn9k_sso_hws_get_work( \
- ws, ev, flags | NIX_RX_MULTI_SEG_F | CPT_RX_WQE_F, \
- ws->lookup_mem); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_ca_seg_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_deq_ca_seg_##name(port, ev, \
- timeout_ticks); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,72 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_deq_tmo_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws *ws = port; \
- uint16_t ret = 1; \
- uint64_t iter; \
- \
- if (ws->swtag_req) { \
- ws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
- return ret; \
- } \
- \
- ret = cn9k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \
- for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \
- ret = cn9k_sso_hws_get_work(ws, ev, flags, \
- ws->lookup_mem); \
- \
- return ret; \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_tmo_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_deq_tmo_##name(port, ev, timeout_ticks); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_tmo_seg_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws *ws = port; \
- uint16_t ret = 1; \
- uint64_t iter; \
- \
- if (ws->swtag_req) { \
- ws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(ws->base + SSOW_LF_GWS_TAG); \
- return ret; \
- } \
- \
- ret = cn9k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \
- for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \
- ret = cn9k_sso_hws_get_work(ws, ev, flags, \
- ws->lookup_mem); \
- \
- return ret; \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_deq_tmo_seg_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_deq_tmo_seg_##name(port, ev, \
- timeout_ticks); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,53 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws_dual *dws = port; \
- uint16_t gw; \
- \
- RTE_SET_USED(timeout_ticks); \
- if (dws->swtag_req) { \
- dws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
- SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- gw = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, flags, \
- dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- return gw; \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_seg_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws_dual *dws = port; \
- uint16_t gw; \
- \
- RTE_SET_USED(timeout_ticks); \
- if (dws->swtag_req) { \
- dws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
- SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- gw = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, flags, \
- dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- return gw; \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,30 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_dual_deq_##name(port, ev, timeout_ticks); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_seg_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_dual_deq_seg_##name(port, ev, \
- timeout_ticks); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,74 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_ca_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws_dual *dws = port; \
- uint16_t gw; \
- \
- RTE_SET_USED(timeout_ticks); \
- if (dws->swtag_req) { \
- dws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
- SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- gw = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, \
- flags | CPT_RX_WQE_F, dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- return gw; \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_ca_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_dual_deq_ca_##name(port, ev, \
- timeout_ticks); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_ca_seg_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws_dual *dws = port; \
- uint16_t gw; \
- \
- RTE_SET_USED(timeout_ticks); \
- if (dws->swtag_req) { \
- dws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
- SSOW_LF_GWS_TAG); \
- return 1; \
- } \
- \
- gw = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, \
- flags | NIX_RX_MULTI_SEG_F | CPT_RX_WQE_F, \
- dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- return gw; \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_ca_seg_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_dual_deq_ca_seg_##name(port, ev, \
- timeout_ticks); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,87 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_worker.h"
-#include "cnxk_eventdev.h"
-#include "cnxk_worker.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws_dual *dws = port; \
- uint16_t ret = 1; \
- uint64_t iter; \
- \
- if (dws->swtag_req) { \
- dws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
- SSOW_LF_GWS_TAG); \
- return ret; \
- } \
- \
- ret = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, flags, \
- dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) { \
- ret = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, \
- flags, dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- } \
- \
- return ret; \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_dual_deq_tmo_##name(port, ev, \
- timeout_ticks); \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_seg_##name( \
- void *port, struct rte_event *ev, uint64_t timeout_ticks) \
- { \
- struct cn9k_sso_hws_dual *dws = port; \
- uint16_t ret = 1; \
- uint64_t iter; \
- \
- if (dws->swtag_req) { \
- dws->swtag_req = 0; \
- cnxk_sso_hws_swtag_wait(dws->base[!dws->vws] + \
- SSOW_LF_GWS_TAG); \
- return ret; \
- } \
- \
- ret = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, flags, \
- dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) { \
- ret = cn9k_sso_hws_dual_get_work( \
- dws->base[dws->vws], dws->base[!dws->vws], ev, \
- flags, dws->lookup_mem, dws->tstamp); \
- dws->vws = !dws->vws; \
- } \
- \
- return ret; \
- } \
- \
- uint16_t __rte_hot cn9k_sso_hws_dual_deq_tmo_seg_burst_##name( \
- void *port, struct rte_event ev[], uint16_t nb_events, \
- uint64_t timeout_ticks) \
- { \
- RTE_SET_USED(nb_events); \
- \
- return cn9k_sso_hws_dual_deq_tmo_seg_##name(port, ev, \
- timeout_ticks); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
@@ -11,14 +11,6 @@ endif
sources = files(
'cn9k_eventdev.c',
'cn9k_worker.c',
- 'cn9k_worker_deq.c',
- 'cn9k_worker_deq_burst.c',
- 'cn9k_worker_deq_ca.c',
- 'cn9k_worker_deq_tmo.c',
- 'cn9k_worker_dual_deq.c',
- 'cn9k_worker_dual_deq_burst.c',
- 'cn9k_worker_dual_deq_ca.c',
- 'cn9k_worker_dual_deq_tmo.c',
'cn9k_worker_tx_enq.c',
'cn9k_worker_tx_enq_seg.c',
'cn9k_worker_dual_tx_enq.c',
@@ -39,6 +31,268 @@ sources = files(
'cnxk_tim_worker.c',
)
+sources += files(
+ 'deq/cn9k/deq_0_15_burst.c',
+ 'deq/cn9k/deq_16_31_burst.c',
+ 'deq/cn9k/deq_32_47_burst.c',
+ 'deq/cn9k/deq_48_63_burst.c',
+ 'deq/cn9k/deq_64_79_burst.c',
+ 'deq/cn9k/deq_80_95_burst.c',
+ 'deq/cn9k/deq_96_111_burst.c',
+ 'deq/cn9k/deq_112_127_burst.c',
+ 'deq/cn9k/deq_0_15_seg_burst.c',
+ 'deq/cn9k/deq_16_31_seg_burst.c',
+ 'deq/cn9k/deq_32_47_seg_burst.c',
+ 'deq/cn9k/deq_48_63_seg_burst.c',
+ 'deq/cn9k/deq_64_79_seg_burst.c',
+ 'deq/cn9k/deq_80_95_seg_burst.c',
+ 'deq/cn9k/deq_96_111_seg_burst.c',
+ 'deq/cn9k/deq_112_127_seg_burst.c',
+ 'deq/cn9k/deq_0_15.c',
+ 'deq/cn9k/deq_16_31.c',
+ 'deq/cn9k/deq_32_47.c',
+ 'deq/cn9k/deq_48_63.c',
+ 'deq/cn9k/deq_64_79.c',
+ 'deq/cn9k/deq_80_95.c',
+ 'deq/cn9k/deq_96_111.c',
+ 'deq/cn9k/deq_112_127.c',
+ 'deq/cn9k/deq_0_15_seg.c',
+ 'deq/cn9k/deq_16_31_seg.c',
+ 'deq/cn9k/deq_32_47_seg.c',
+ 'deq/cn9k/deq_48_63_seg.c',
+ 'deq/cn9k/deq_64_79_seg.c',
+ 'deq/cn9k/deq_80_95_seg.c',
+ 'deq/cn9k/deq_96_111_seg.c',
+ 'deq/cn9k/deq_112_127_seg.c',
+ 'deq/cn9k/deq_0_15_tmo.c',
+ 'deq/cn9k/deq_16_31_tmo.c',
+ 'deq/cn9k/deq_32_47_tmo.c',
+ 'deq/cn9k/deq_48_63_tmo.c',
+ 'deq/cn9k/deq_64_79_tmo.c',
+ 'deq/cn9k/deq_80_95_tmo.c',
+ 'deq/cn9k/deq_96_111_tmo.c',
+ 'deq/cn9k/deq_112_127_tmo.c',
+ 'deq/cn9k/deq_0_15_tmo_burst.c',
+ 'deq/cn9k/deq_16_31_tmo_burst.c',
+ 'deq/cn9k/deq_32_47_tmo_burst.c',
+ 'deq/cn9k/deq_48_63_tmo_burst.c',
+ 'deq/cn9k/deq_64_79_tmo_burst.c',
+ 'deq/cn9k/deq_80_95_tmo_burst.c',
+ 'deq/cn9k/deq_96_111_tmo_burst.c',
+ 'deq/cn9k/deq_112_127_tmo_burst.c',
+ 'deq/cn9k/deq_0_15_tmo_seg.c',
+ 'deq/cn9k/deq_16_31_tmo_seg.c',
+ 'deq/cn9k/deq_32_47_tmo_seg.c',
+ 'deq/cn9k/deq_48_63_tmo_seg.c',
+ 'deq/cn9k/deq_64_79_tmo_seg.c',
+ 'deq/cn9k/deq_80_95_tmo_seg.c',
+ 'deq/cn9k/deq_96_111_tmo_seg.c',
+ 'deq/cn9k/deq_112_127_tmo_seg.c',
+ 'deq/cn9k/deq_0_15_tmo_seg_burst.c',
+ 'deq/cn9k/deq_16_31_tmo_seg_burst.c',
+ 'deq/cn9k/deq_32_47_tmo_seg_burst.c',
+ 'deq/cn9k/deq_48_63_tmo_seg_burst.c',
+ 'deq/cn9k/deq_64_79_tmo_seg_burst.c',
+ 'deq/cn9k/deq_80_95_tmo_seg_burst.c',
+ 'deq/cn9k/deq_96_111_tmo_seg_burst.c',
+ 'deq/cn9k/deq_112_127_tmo_seg_burst.c',
+ 'deq/cn9k/deq_0_15_ca.c',
+ 'deq/cn9k/deq_16_31_ca.c',
+ 'deq/cn9k/deq_32_47_ca.c',
+ 'deq/cn9k/deq_48_63_ca.c',
+ 'deq/cn9k/deq_64_79_ca.c',
+ 'deq/cn9k/deq_80_95_ca.c',
+ 'deq/cn9k/deq_96_111_ca.c',
+ 'deq/cn9k/deq_112_127_ca.c',
+ 'deq/cn9k/deq_0_15_ca_burst.c',
+ 'deq/cn9k/deq_16_31_ca_burst.c',
+ 'deq/cn9k/deq_32_47_ca_burst.c',
+ 'deq/cn9k/deq_48_63_ca_burst.c',
+ 'deq/cn9k/deq_64_79_ca_burst.c',
+ 'deq/cn9k/deq_80_95_ca_burst.c',
+ 'deq/cn9k/deq_96_111_ca_burst.c',
+ 'deq/cn9k/deq_112_127_ca_burst.c',
+ 'deq/cn9k/deq_0_15_ca_seg.c',
+ 'deq/cn9k/deq_16_31_ca_seg.c',
+ 'deq/cn9k/deq_32_47_ca_seg.c',
+ 'deq/cn9k/deq_48_63_ca_seg.c',
+ 'deq/cn9k/deq_64_79_ca_seg.c',
+ 'deq/cn9k/deq_80_95_ca_seg.c',
+ 'deq/cn9k/deq_96_111_ca_seg.c',
+ 'deq/cn9k/deq_112_127_ca_seg.c',
+ 'deq/cn9k/deq_0_15_ca_seg_burst.c',
+ 'deq/cn9k/deq_16_31_ca_seg_burst.c',
+ 'deq/cn9k/deq_32_47_ca_seg_burst.c',
+ 'deq/cn9k/deq_48_63_ca_seg_burst.c',
+ 'deq/cn9k/deq_64_79_ca_seg_burst.c',
+ 'deq/cn9k/deq_80_95_ca_seg_burst.c',
+ 'deq/cn9k/deq_96_111_ca_seg_burst.c',
+ 'deq/cn9k/deq_112_127_ca_seg_burst.c',
+ 'deq/cn9k/deq_0_15_ca_tmo.c',
+ 'deq/cn9k/deq_16_31_ca_tmo.c',
+ 'deq/cn9k/deq_32_47_ca_tmo.c',
+ 'deq/cn9k/deq_48_63_ca_tmo.c',
+ 'deq/cn9k/deq_64_79_ca_tmo.c',
+ 'deq/cn9k/deq_80_95_ca_tmo.c',
+ 'deq/cn9k/deq_96_111_ca_tmo.c',
+ 'deq/cn9k/deq_112_127_ca_tmo.c',
+ 'deq/cn9k/deq_0_15_ca_tmo_burst.c',
+ 'deq/cn9k/deq_16_31_ca_tmo_burst.c',
+ 'deq/cn9k/deq_32_47_ca_tmo_burst.c',
+ 'deq/cn9k/deq_48_63_ca_tmo_burst.c',
+ 'deq/cn9k/deq_64_79_ca_tmo_burst.c',
+ 'deq/cn9k/deq_80_95_ca_tmo_burst.c',
+ 'deq/cn9k/deq_96_111_ca_tmo_burst.c',
+ 'deq/cn9k/deq_112_127_ca_tmo_burst.c',
+ 'deq/cn9k/deq_0_15_ca_tmo_seg.c',
+ 'deq/cn9k/deq_16_31_ca_tmo_seg.c',
+ 'deq/cn9k/deq_32_47_ca_tmo_seg.c',
+ 'deq/cn9k/deq_48_63_ca_tmo_seg.c',
+ 'deq/cn9k/deq_64_79_ca_tmo_seg.c',
+ 'deq/cn9k/deq_80_95_ca_tmo_seg.c',
+ 'deq/cn9k/deq_96_111_ca_tmo_seg.c',
+ 'deq/cn9k/deq_112_127_ca_tmo_seg.c',
+ 'deq/cn9k/deq_0_15_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_16_31_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_32_47_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_48_63_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_64_79_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_80_95_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_96_111_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_112_127_ca_tmo_seg_burst.c',
+)
+
+sources += files(
+ 'deq/cn9k/deq_0_15_dual_burst.c',
+ 'deq/cn9k/deq_16_31_dual_burst.c',
+ 'deq/cn9k/deq_32_47_dual_burst.c',
+ 'deq/cn9k/deq_48_63_dual_burst.c',
+ 'deq/cn9k/deq_64_79_dual_burst.c',
+ 'deq/cn9k/deq_80_95_dual_burst.c',
+ 'deq/cn9k/deq_96_111_dual_burst.c',
+ 'deq/cn9k/deq_112_127_dual_burst.c',
+ 'deq/cn9k/deq_0_15_dual_seg_burst.c',
+ 'deq/cn9k/deq_16_31_dual_seg_burst.c',
+ 'deq/cn9k/deq_32_47_dual_seg_burst.c',
+ 'deq/cn9k/deq_48_63_dual_seg_burst.c',
+ 'deq/cn9k/deq_64_79_dual_seg_burst.c',
+ 'deq/cn9k/deq_80_95_dual_seg_burst.c',
+ 'deq/cn9k/deq_96_111_dual_seg_burst.c',
+ 'deq/cn9k/deq_112_127_dual_seg_burst.c',
+ 'deq/cn9k/deq_0_15_dual.c',
+ 'deq/cn9k/deq_16_31_dual.c',
+ 'deq/cn9k/deq_32_47_dual.c',
+ 'deq/cn9k/deq_48_63_dual.c',
+ 'deq/cn9k/deq_64_79_dual.c',
+ 'deq/cn9k/deq_80_95_dual.c',
+ 'deq/cn9k/deq_96_111_dual.c',
+ 'deq/cn9k/deq_112_127_dual.c',
+ 'deq/cn9k/deq_0_15_dual_seg.c',
+ 'deq/cn9k/deq_16_31_dual_seg.c',
+ 'deq/cn9k/deq_32_47_dual_seg.c',
+ 'deq/cn9k/deq_48_63_dual_seg.c',
+ 'deq/cn9k/deq_64_79_dual_seg.c',
+ 'deq/cn9k/deq_80_95_dual_seg.c',
+ 'deq/cn9k/deq_96_111_dual_seg.c',
+ 'deq/cn9k/deq_112_127_dual_seg.c',
+ 'deq/cn9k/deq_0_15_dual_tmo.c',
+ 'deq/cn9k/deq_16_31_dual_tmo.c',
+ 'deq/cn9k/deq_32_47_dual_tmo.c',
+ 'deq/cn9k/deq_48_63_dual_tmo.c',
+ 'deq/cn9k/deq_64_79_dual_tmo.c',
+ 'deq/cn9k/deq_80_95_dual_tmo.c',
+ 'deq/cn9k/deq_96_111_dual_tmo.c',
+ 'deq/cn9k/deq_112_127_dual_tmo.c',
+ 'deq/cn9k/deq_0_15_dual_tmo_burst.c',
+ 'deq/cn9k/deq_16_31_dual_tmo_burst.c',
+ 'deq/cn9k/deq_32_47_dual_tmo_burst.c',
+ 'deq/cn9k/deq_48_63_dual_tmo_burst.c',
+ 'deq/cn9k/deq_64_79_dual_tmo_burst.c',
+ 'deq/cn9k/deq_80_95_dual_tmo_burst.c',
+ 'deq/cn9k/deq_96_111_dual_tmo_burst.c',
+ 'deq/cn9k/deq_112_127_dual_tmo_burst.c',
+ 'deq/cn9k/deq_0_15_dual_tmo_seg.c',
+ 'deq/cn9k/deq_16_31_dual_tmo_seg.c',
+ 'deq/cn9k/deq_32_47_dual_tmo_seg.c',
+ 'deq/cn9k/deq_48_63_dual_tmo_seg.c',
+ 'deq/cn9k/deq_64_79_dual_tmo_seg.c',
+ 'deq/cn9k/deq_80_95_dual_tmo_seg.c',
+ 'deq/cn9k/deq_96_111_dual_tmo_seg.c',
+ 'deq/cn9k/deq_112_127_dual_tmo_seg.c',
+ 'deq/cn9k/deq_0_15_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_16_31_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_32_47_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_48_63_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_64_79_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_80_95_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_96_111_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_112_127_dual_tmo_seg_burst.c',
+ 'deq/cn9k/deq_0_15_dual_ca.c',
+ 'deq/cn9k/deq_16_31_dual_ca.c',
+ 'deq/cn9k/deq_32_47_dual_ca.c',
+ 'deq/cn9k/deq_48_63_dual_ca.c',
+ 'deq/cn9k/deq_64_79_dual_ca.c',
+ 'deq/cn9k/deq_80_95_dual_ca.c',
+ 'deq/cn9k/deq_96_111_dual_ca.c',
+ 'deq/cn9k/deq_112_127_dual_ca.c',
+ 'deq/cn9k/deq_0_15_dual_ca_burst.c',
+ 'deq/cn9k/deq_16_31_dual_ca_burst.c',
+ 'deq/cn9k/deq_32_47_dual_ca_burst.c',
+ 'deq/cn9k/deq_48_63_dual_ca_burst.c',
+ 'deq/cn9k/deq_64_79_dual_ca_burst.c',
+ 'deq/cn9k/deq_80_95_dual_ca_burst.c',
+ 'deq/cn9k/deq_96_111_dual_ca_burst.c',
+ 'deq/cn9k/deq_112_127_dual_ca_burst.c',
+ 'deq/cn9k/deq_0_15_dual_ca_seg.c',
+ 'deq/cn9k/deq_16_31_dual_ca_seg.c',
+ 'deq/cn9k/deq_32_47_dual_ca_seg.c',
+ 'deq/cn9k/deq_48_63_dual_ca_seg.c',
+ 'deq/cn9k/deq_64_79_dual_ca_seg.c',
+ 'deq/cn9k/deq_80_95_dual_ca_seg.c',
+ 'deq/cn9k/deq_96_111_dual_ca_seg.c',
+ 'deq/cn9k/deq_112_127_dual_ca_seg.c',
+ 'deq/cn9k/deq_0_15_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_16_31_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_32_47_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_48_63_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_64_79_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_80_95_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_96_111_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_112_127_dual_ca_seg_burst.c',
+ 'deq/cn9k/deq_0_15_dual_ca_tmo.c',
+ 'deq/cn9k/deq_16_31_dual_ca_tmo.c',
+ 'deq/cn9k/deq_32_47_dual_ca_tmo.c',
+ 'deq/cn9k/deq_48_63_dual_ca_tmo.c',
+ 'deq/cn9k/deq_64_79_dual_ca_tmo.c',
+ 'deq/cn9k/deq_80_95_dual_ca_tmo.c',
+ 'deq/cn9k/deq_96_111_dual_ca_tmo.c',
+ 'deq/cn9k/deq_112_127_dual_ca_tmo.c',
+ 'deq/cn9k/deq_0_15_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_16_31_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_32_47_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_48_63_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_64_79_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_80_95_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_96_111_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_112_127_dual_ca_tmo_burst.c',
+ 'deq/cn9k/deq_0_15_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_16_31_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_32_47_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_48_63_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_64_79_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_80_95_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_96_111_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_112_127_dual_ca_tmo_seg.c',
+ 'deq/cn9k/deq_0_15_dual_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_16_31_dual_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_32_47_dual_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_48_63_dual_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_64_79_dual_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_80_95_dual_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_96_111_dual_ca_tmo_seg_burst.c',
+ 'deq/cn9k/deq_112_127_dual_ca_tmo_seg_burst.c',
+)
+
extra_flags = ['-flax-vector-conversions', '-Wno-strict-aliasing']
foreach flag: extra_flags
if cc.has_argument(flag)
@@ -18,6 +18,7 @@
#define NIX_RX_OFFLOAD_TSTAMP_F BIT(4)
#define NIX_RX_OFFLOAD_VLAN_STRIP_F BIT(5)
#define NIX_RX_OFFLOAD_SECURITY_F BIT(6)
+#define NIX_RX_OFFLOAD_MAX (NIX_RX_OFFLOAD_SECURITY_F << 1)
/* Flags to control cqe_to_mbuf conversion function.
* Defining it from backwards to denote its been
@@ -853,278 +854,214 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts,
#define R_SEC_F NIX_RX_OFFLOAD_SECURITY_F
/* [R_SEC_F] [RX_VLAN_F] [TS] [MARK] [CKSUM] [PTYPE] [RSS] */
-#define NIX_RX_FASTPATH_MODES \
-R(no_offload, 0, 0, 0, 0, 0, 0, 0, \
- NIX_RX_OFFLOAD_NONE) \
-R(rss, 0, 0, 0, 0, 0, 0, 1, \
- RSS_F) \
-R(ptype, 0, 0, 0, 0, 0, 1, 0, \
- PTYPE_F) \
-R(ptype_rss, 0, 0, 0, 0, 0, 1, 1, \
- PTYPE_F | RSS_F) \
-R(cksum, 0, 0, 0, 0, 1, 0, 0, \
- CKSUM_F) \
-R(cksum_rss, 0, 0, 0, 0, 1, 0, 1, \
- CKSUM_F | RSS_F) \
-R(cksum_ptype, 0, 0, 0, 0, 1, 1, 0, \
- CKSUM_F | PTYPE_F) \
-R(cksum_ptype_rss, 0, 0, 0, 0, 1, 1, 1, \
- CKSUM_F | PTYPE_F | RSS_F) \
-R(mark, 0, 0, 0, 1, 0, 0, 0, \
- MARK_F) \
-R(mark_rss, 0, 0, 0, 1, 0, 0, 1, \
- MARK_F | RSS_F) \
-R(mark_ptype, 0, 0, 0, 1, 0, 1, 0, \
- MARK_F | PTYPE_F) \
-R(mark_ptype_rss, 0, 0, 0, 1, 0, 1, 1, \
- MARK_F | PTYPE_F | RSS_F) \
-R(mark_cksum, 0, 0, 0, 1, 1, 0, 0, \
- MARK_F | CKSUM_F) \
-R(mark_cksum_rss, 0, 0, 0, 1, 1, 0, 1, \
- MARK_F | CKSUM_F | RSS_F) \
-R(mark_cksum_ptype, 0, 0, 0, 1, 1, 1, 0, \
- MARK_F | CKSUM_F | PTYPE_F) \
-R(mark_cksum_ptype_rss, 0, 0, 0, 1, 1, 1, 1, \
- MARK_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(ts, 0, 0, 1, 0, 0, 0, 0, \
- TS_F) \
-R(ts_rss, 0, 0, 1, 0, 0, 0, 1, \
- TS_F | RSS_F) \
-R(ts_ptype, 0, 0, 1, 0, 0, 1, 0, \
- TS_F | PTYPE_F) \
-R(ts_ptype_rss, 0, 0, 1, 0, 0, 1, 1, \
- TS_F | PTYPE_F | RSS_F) \
-R(ts_cksum, 0, 0, 1, 0, 1, 0, 0, \
- TS_F | CKSUM_F) \
-R(ts_cksum_rss, 0, 0, 1, 0, 1, 0, 1, \
- TS_F | CKSUM_F | RSS_F) \
-R(ts_cksum_ptype, 0, 0, 1, 0, 1, 1, 0, \
- TS_F | CKSUM_F | PTYPE_F) \
-R(ts_cksum_ptype_rss, 0, 0, 1, 0, 1, 1, 1, \
- TS_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(ts_mark, 0, 0, 1, 1, 0, 0, 0, \
- TS_F | MARK_F) \
-R(ts_mark_rss, 0, 0, 1, 1, 0, 0, 1, \
- TS_F | MARK_F | RSS_F) \
-R(ts_mark_ptype, 0, 0, 1, 1, 0, 1, 0, \
- TS_F | MARK_F | PTYPE_F) \
-R(ts_mark_ptype_rss, 0, 0, 1, 1, 0, 1, 1, \
- TS_F | MARK_F | PTYPE_F | RSS_F) \
-R(ts_mark_cksum, 0, 0, 1, 1, 1, 0, 0, \
- TS_F | MARK_F | CKSUM_F) \
-R(ts_mark_cksum_rss, 0, 0, 1, 1, 1, 0, 1, \
- TS_F | MARK_F | CKSUM_F | RSS_F) \
-R(ts_mark_cksum_ptype, 0, 0, 1, 1, 1, 1, 0, \
- TS_F | MARK_F | CKSUM_F | PTYPE_F) \
-R(ts_mark_cksum_ptype_rss, 0, 0, 1, 1, 1, 1, 1, \
- TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(vlan, 0, 1, 0, 0, 0, 0, 0, \
- RX_VLAN_F) \
-R(vlan_rss, 0, 1, 0, 0, 0, 0, 1, \
- RX_VLAN_F | RSS_F) \
-R(vlan_ptype, 0, 1, 0, 0, 0, 1, 0, \
- RX_VLAN_F | PTYPE_F) \
-R(vlan_ptype_rss, 0, 1, 0, 0, 0, 1, 1, \
- RX_VLAN_F | PTYPE_F | RSS_F) \
-R(vlan_cksum, 0, 1, 0, 0, 1, 0, 0, \
- RX_VLAN_F | CKSUM_F) \
-R(vlan_cksum_rss, 0, 1, 0, 0, 1, 0, 1, \
- RX_VLAN_F | CKSUM_F | RSS_F) \
-R(vlan_cksum_ptype, 0, 1, 0, 0, 1, 1, 0, \
- RX_VLAN_F | CKSUM_F | PTYPE_F) \
-R(vlan_cksum_ptype_rss, 0, 1, 0, 0, 1, 1, 1, \
- RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(vlan_mark, 0, 1, 0, 1, 0, 0, 0, \
- RX_VLAN_F | MARK_F) \
-R(vlan_mark_rss, 0, 1, 0, 1, 0, 0, 1, \
- RX_VLAN_F | MARK_F | RSS_F) \
-R(vlan_mark_ptype, 0, 1, 0, 1, 0, 1, 0, \
- RX_VLAN_F | MARK_F | PTYPE_F) \
-R(vlan_mark_ptype_rss, 0, 1, 0, 1, 0, 1, 1, \
- RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \
-R(vlan_mark_cksum, 0, 1, 0, 1, 1, 0, 0, \
- RX_VLAN_F | MARK_F | CKSUM_F) \
-R(vlan_mark_cksum_rss, 0, 1, 0, 1, 1, 0, 1, \
- RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \
-R(vlan_mark_cksum_ptype, 0, 1, 0, 1, 1, 1, 0, \
- RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \
-R(vlan_mark_cksum_ptype_rss, 0, 1, 0, 1, 1, 1, 1, \
- RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(vlan_ts, 0, 1, 1, 0, 0, 0, 0, \
- RX_VLAN_F | TS_F) \
-R(vlan_ts_rss, 0, 1, 1, 0, 0, 0, 1, \
- RX_VLAN_F | TS_F | RSS_F) \
-R(vlan_ts_ptype, 0, 1, 1, 0, 0, 1, 0, \
- RX_VLAN_F | TS_F | PTYPE_F) \
-R(vlan_ts_ptype_rss, 0, 1, 1, 0, 0, 1, 1, \
- RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
-R(vlan_ts_cksum, 0, 1, 1, 0, 1, 0, 0, \
- RX_VLAN_F | TS_F | CKSUM_F) \
-R(vlan_ts_cksum_rss, 0, 1, 1, 0, 1, 0, 1, \
- RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
-R(vlan_ts_cksum_ptype, 0, 1, 1, 0, 1, 1, 0, \
- RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \
-R(vlan_ts_cksum_ptype_rss, 0, 1, 1, 0, 1, 1, 1, \
- RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(vlan_ts_mark, 0, 1, 1, 1, 0, 0, 0, \
- RX_VLAN_F | TS_F | MARK_F) \
-R(vlan_ts_mark_rss, 0, 1, 1, 1, 0, 0, 1, \
- RX_VLAN_F | TS_F | MARK_F | RSS_F) \
-R(vlan_ts_mark_ptype, 0, 1, 1, 1, 0, 1, 0, \
- RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \
-R(vlan_ts_mark_ptype_rss, 0, 1, 1, 1, 0, 1, 1, \
- RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
-R(vlan_ts_mark_cksum, 0, 1, 1, 1, 1, 0, 0, \
- RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \
-R(vlan_ts_mark_cksum_rss, 0, 1, 1, 1, 1, 0, 1, \
- RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
-R(vlan_ts_mark_cksum_ptype, 0, 1, 1, 1, 1, 1, 0, \
- RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
-R(vlan_ts_mark_cksum_ptype_rss, 0, 1, 1, 1, 1, 1, 1, \
- RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec, 1, 0, 0, 0, 0, 0, 0, \
- R_SEC_F) \
-R(sec_rss, 1, 0, 0, 0, 0, 0, 1, \
- RSS_F) \
-R(sec_ptype, 1, 0, 0, 0, 0, 1, 0, \
- R_SEC_F | PTYPE_F) \
-R(sec_ptype_rss, 1, 0, 0, 0, 0, 1, 1, \
- R_SEC_F | PTYPE_F | RSS_F) \
-R(sec_cksum, 1, 0, 0, 0, 1, 0, 0, \
- R_SEC_F | CKSUM_F) \
-R(sec_cksum_rss, 1, 0, 0, 0, 1, 0, 1, \
- R_SEC_F | CKSUM_F | RSS_F) \
-R(sec_cksum_ptype, 1, 0, 0, 0, 1, 1, 0, \
- R_SEC_F | CKSUM_F | PTYPE_F) \
-R(sec_cksum_ptype_rss, 1, 0, 0, 0, 1, 1, 1, \
- R_SEC_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec_mark, 1, 0, 0, 1, 0, 0, 0, \
- R_SEC_F | MARK_F) \
-R(sec_mark_rss, 1, 0, 0, 1, 0, 0, 1, \
- R_SEC_F | MARK_F | RSS_F) \
-R(sec_mark_ptype, 1, 0, 0, 1, 0, 1, 0, \
- R_SEC_F | MARK_F | PTYPE_F) \
-R(sec_mark_ptype_rss, 1, 0, 0, 1, 0, 1, 1, \
- R_SEC_F | MARK_F | PTYPE_F | RSS_F) \
-R(sec_mark_cksum, 1, 0, 0, 1, 1, 0, 0, \
- R_SEC_F | MARK_F | CKSUM_F) \
-R(sec_mark_cksum_rss, 1, 0, 0, 1, 1, 0, 1, \
- R_SEC_F | MARK_F | CKSUM_F | RSS_F) \
-R(sec_mark_cksum_ptype, 1, 0, 0, 1, 1, 1, 0, \
- R_SEC_F | MARK_F | CKSUM_F | PTYPE_F) \
-R(sec_mark_cksum_ptype_rss, 1, 0, 0, 1, 1, 1, 1, \
- R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec_ts, 1, 0, 1, 0, 0, 0, 0, \
- R_SEC_F | TS_F) \
-R(sec_ts_rss, 1, 0, 1, 0, 0, 0, 1, \
- R_SEC_F | TS_F | RSS_F) \
-R(sec_ts_ptype, 1, 0, 1, 0, 0, 1, 0, \
- R_SEC_F | TS_F | PTYPE_F) \
-R(sec_ts_ptype_rss, 1, 0, 1, 0, 0, 1, 1, \
- R_SEC_F | TS_F | PTYPE_F | RSS_F) \
-R(sec_ts_cksum, 1, 0, 1, 0, 1, 0, 0, \
- R_SEC_F | TS_F | CKSUM_F) \
-R(sec_ts_cksum_rss, 1, 0, 1, 0, 1, 0, 1, \
- R_SEC_F | TS_F | CKSUM_F | RSS_F) \
-R(sec_ts_cksum_ptype, 1, 0, 1, 0, 1, 1, 0, \
- R_SEC_F | TS_F | CKSUM_F | PTYPE_F) \
-R(sec_ts_cksum_ptype_rss, 1, 0, 1, 0, 1, 1, 1, \
- R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec_ts_mark, 1, 0, 1, 1, 0, 0, 0, \
- R_SEC_F | TS_F | MARK_F) \
-R(sec_ts_mark_rss, 1, 0, 1, 1, 0, 0, 1, \
- R_SEC_F | TS_F | MARK_F | RSS_F) \
-R(sec_ts_mark_ptype, 1, 0, 1, 1, 0, 1, 0, \
- R_SEC_F | TS_F | MARK_F | PTYPE_F) \
-R(sec_ts_mark_ptype_rss, 1, 0, 1, 1, 0, 1, 1, \
- R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
-R(sec_ts_mark_cksum, 1, 0, 1, 1, 1, 0, 0, \
- R_SEC_F | TS_F | MARK_F | CKSUM_F) \
-R(sec_ts_mark_cksum_rss, 1, 0, 1, 1, 1, 0, 1, \
- R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
-R(sec_ts_mark_cksum_ptype, 1, 0, 1, 1, 1, 1, 0, \
- R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
-R(sec_ts_mark_cksum_ptype_rss, 1, 0, 1, 1, 1, 1, 1, \
- R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec_vlan, 1, 1, 0, 0, 0, 0, 0, \
- R_SEC_F | RX_VLAN_F) \
-R(sec_vlan_rss, 1, 1, 0, 0, 0, 0, 1, \
- R_SEC_F | RX_VLAN_F | RSS_F) \
-R(sec_vlan_ptype, 1, 1, 0, 0, 0, 1, 0, \
- R_SEC_F | RX_VLAN_F | PTYPE_F) \
-R(sec_vlan_ptype_rss, 1, 1, 0, 0, 0, 1, 1, \
- R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F) \
-R(sec_vlan_cksum, 1, 1, 0, 0, 1, 0, 0, \
- R_SEC_F | RX_VLAN_F | CKSUM_F) \
-R(sec_vlan_cksum_rss, 1, 1, 0, 0, 1, 0, 1, \
- R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F) \
-R(sec_vlan_cksum_ptype, 1, 1, 0, 0, 1, 1, 0, \
- R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F) \
-R(sec_vlan_cksum_ptype_rss, 1, 1, 0, 0, 1, 1, 1, \
- R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec_vlan_mark, 1, 1, 0, 1, 0, 0, 0, \
- R_SEC_F | RX_VLAN_F | MARK_F) \
-R(sec_vlan_mark_rss, 1, 1, 0, 1, 0, 0, 1, \
- R_SEC_F | RX_VLAN_F | MARK_F | RSS_F) \
-R(sec_vlan_mark_ptype, 1, 1, 0, 1, 0, 1, 0, \
- R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F) \
-R(sec_vlan_mark_ptype_rss, 1, 1, 0, 1, 0, 1, 1, \
- R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \
-R(sec_vlan_mark_cksum, 1, 1, 0, 1, 1, 0, 0, \
- R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F) \
-R(sec_vlan_mark_cksum_rss, 1, 1, 0, 1, 1, 0, 1, \
- R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \
-R(sec_vlan_mark_cksum_ptype, 1, 1, 0, 1, 1, 1, 0, \
- R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \
-R(sec_vlan_mark_cksum_ptype_rss, 1, 1, 0, 1, 1, 1, 1, \
- R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec_vlan_ts, 1, 1, 1, 0, 0, 0, 0, \
- R_SEC_F | RX_VLAN_F | TS_F) \
-R(sec_vlan_ts_rss, 1, 1, 1, 0, 0, 0, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | RSS_F) \
-R(sec_vlan_ts_ptype, 1, 1, 1, 0, 0, 1, 0, \
- R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F) \
-R(sec_vlan_ts_ptype_rss, 1, 1, 1, 0, 0, 1, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
-R(sec_vlan_ts_cksum, 1, 1, 1, 0, 1, 0, 0, \
- R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F) \
-R(sec_vlan_ts_cksum_rss, 1, 1, 1, 0, 1, 0, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
-R(sec_vlan_ts_cksum_ptype, 1, 1, 1, 0, 1, 1, 0, \
- R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \
-R(sec_vlan_ts_cksum_ptype_rss, 1, 1, 1, 0, 1, 1, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
-R(sec_vlan_ts_mark, 1, 1, 1, 1, 0, 0, 0, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F) \
-R(sec_vlan_ts_mark_rss, 1, 1, 1, 1, 0, 0, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F) \
-R(sec_vlan_ts_mark_ptype, 1, 1, 1, 1, 0, 1, 0, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \
-R(sec_vlan_ts_mark_ptype_rss, 1, 1, 1, 1, 0, 1, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
-R(sec_vlan_ts_mark_cksum, 1, 1, 1, 1, 1, 0, 0, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \
-R(sec_vlan_ts_mark_cksum_rss, 1, 1, 1, 1, 1, 0, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
-R(sec_vlan_ts_mark_cksum_ptype, 1, 1, 1, 1, 1, 1, 0, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
-R(sec_vlan_ts_mark_cksum_ptype_rss, 1, 1, 1, 1, 1, 1, 1, \
- R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
+#define NIX_RX_FASTPATH_MODES_0_15 \
+ R(no_offload, NIX_RX_OFFLOAD_NONE) \
+ R(rss, RSS_F) \
+ R(ptype, PTYPE_F) \
+ R(ptype_rss, PTYPE_F | RSS_F) \
+ R(cksum, CKSUM_F) \
+ R(cksum_rss, CKSUM_F | RSS_F) \
+ R(cksum_ptype, CKSUM_F | PTYPE_F) \
+ R(cksum_ptype_rss, CKSUM_F | PTYPE_F | RSS_F) \
+ R(mark, MARK_F) \
+ R(mark_rss, MARK_F | RSS_F) \
+ R(mark_ptype, MARK_F | PTYPE_F) \
+ R(mark_ptype_rss, MARK_F | PTYPE_F | RSS_F) \
+ R(mark_cksum, MARK_F | CKSUM_F) \
+ R(mark_cksum_rss, MARK_F | CKSUM_F | RSS_F) \
+ R(mark_cksum_ptype, MARK_F | CKSUM_F | PTYPE_F) \
+ R(mark_cksum_ptype_rss, MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_16_31 \
+ R(ts, TS_F) \
+ R(ts_rss, TS_F | RSS_F) \
+ R(ts_ptype, TS_F | PTYPE_F) \
+ R(ts_ptype_rss, TS_F | PTYPE_F | RSS_F) \
+ R(ts_cksum, TS_F | CKSUM_F) \
+ R(ts_cksum_rss, TS_F | CKSUM_F | RSS_F) \
+ R(ts_cksum_ptype, TS_F | CKSUM_F | PTYPE_F) \
+ R(ts_cksum_ptype_rss, TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(ts_mark, TS_F | MARK_F) \
+ R(ts_mark_rss, TS_F | MARK_F | RSS_F) \
+ R(ts_mark_ptype, TS_F | MARK_F | PTYPE_F) \
+ R(ts_mark_ptype_rss, TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(ts_mark_cksum, TS_F | MARK_F | CKSUM_F) \
+ R(ts_mark_cksum_rss, TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(ts_mark_cksum_ptype, TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(ts_mark_cksum_ptype_rss, TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_32_47 \
+ R(vlan, RX_VLAN_F) \
+ R(vlan_rss, RX_VLAN_F | RSS_F) \
+ R(vlan_ptype, RX_VLAN_F | PTYPE_F) \
+ R(vlan_ptype_rss, RX_VLAN_F | PTYPE_F | RSS_F) \
+ R(vlan_cksum, RX_VLAN_F | CKSUM_F) \
+ R(vlan_cksum_rss, RX_VLAN_F | CKSUM_F | RSS_F) \
+ R(vlan_cksum_ptype, RX_VLAN_F | CKSUM_F | PTYPE_F) \
+ R(vlan_cksum_ptype_rss, RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(vlan_mark, RX_VLAN_F | MARK_F) \
+ R(vlan_mark_rss, RX_VLAN_F | MARK_F | RSS_F) \
+ R(vlan_mark_ptype, RX_VLAN_F | MARK_F | PTYPE_F) \
+ R(vlan_mark_ptype_rss, RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \
+ R(vlan_mark_cksum, RX_VLAN_F | MARK_F | CKSUM_F) \
+ R(vlan_mark_cksum_rss, RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \
+ R(vlan_mark_cksum_ptype, RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(vlan_mark_cksum_ptype_rss, \
+ RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_48_63 \
+ R(vlan_ts, RX_VLAN_F | TS_F) \
+ R(vlan_ts_rss, RX_VLAN_F | TS_F | RSS_F) \
+ R(vlan_ts_ptype, RX_VLAN_F | TS_F | PTYPE_F) \
+ R(vlan_ts_ptype_rss, RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
+ R(vlan_ts_cksum, RX_VLAN_F | TS_F | CKSUM_F) \
+ R(vlan_ts_cksum_rss, RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
+ R(vlan_ts_cksum_ptype, RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \
+ R(vlan_ts_cksum_ptype_rss, \
+ RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(vlan_ts_mark, RX_VLAN_F | TS_F | MARK_F) \
+ R(vlan_ts_mark_rss, RX_VLAN_F | TS_F | MARK_F | RSS_F) \
+ R(vlan_ts_mark_ptype, RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \
+ R(vlan_ts_mark_ptype_rss, RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(vlan_ts_mark_cksum, RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \
+ R(vlan_ts_mark_cksum_rss, RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(vlan_ts_mark_cksum_ptype, \
+ RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(vlan_ts_mark_cksum_ptype_rss, \
+ RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_64_79 \
+ R(sec, R_SEC_F) \
+ R(sec_rss, R_SEC_F | RSS_F) \
+ R(sec_ptype, R_SEC_F | PTYPE_F) \
+ R(sec_ptype_rss, R_SEC_F | PTYPE_F | RSS_F) \
+ R(sec_cksum, R_SEC_F | CKSUM_F) \
+ R(sec_cksum_rss, R_SEC_F | CKSUM_F | RSS_F) \
+ R(sec_cksum_ptype, R_SEC_F | CKSUM_F | PTYPE_F) \
+ R(sec_cksum_ptype_rss, R_SEC_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_mark, R_SEC_F | MARK_F) \
+ R(sec_mark_rss, R_SEC_F | MARK_F | RSS_F) \
+ R(sec_mark_ptype, R_SEC_F | MARK_F | PTYPE_F) \
+ R(sec_mark_ptype_rss, R_SEC_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_mark_cksum, R_SEC_F | MARK_F | CKSUM_F) \
+ R(sec_mark_cksum_rss, R_SEC_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_mark_cksum_ptype, R_SEC_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_mark_cksum_ptype_rss, \
+ R_SEC_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_80_95 \
+ R(sec_ts, R_SEC_F | TS_F) \
+ R(sec_ts_rss, R_SEC_F | TS_F | RSS_F) \
+ R(sec_ts_ptype, R_SEC_F | TS_F | PTYPE_F) \
+ R(sec_ts_ptype_rss, R_SEC_F | TS_F | PTYPE_F | RSS_F) \
+ R(sec_ts_cksum, R_SEC_F | TS_F | CKSUM_F) \
+ R(sec_ts_cksum_rss, R_SEC_F | TS_F | CKSUM_F | RSS_F) \
+ R(sec_ts_cksum_ptype, R_SEC_F | TS_F | CKSUM_F | PTYPE_F) \
+ R(sec_ts_cksum_ptype_rss, R_SEC_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_ts_mark, R_SEC_F | TS_F | MARK_F) \
+ R(sec_ts_mark_rss, R_SEC_F | TS_F | MARK_F | RSS_F) \
+ R(sec_ts_mark_ptype, R_SEC_F | TS_F | MARK_F | PTYPE_F) \
+ R(sec_ts_mark_ptype_rss, R_SEC_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_ts_mark_cksum, R_SEC_F | TS_F | MARK_F | CKSUM_F) \
+ R(sec_ts_mark_cksum_rss, R_SEC_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_ts_mark_cksum_ptype, \
+ R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_ts_mark_cksum_ptype_rss, \
+ R_SEC_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_96_111 \
+ R(sec_vlan, R_SEC_F | RX_VLAN_F) \
+ R(sec_vlan_rss, R_SEC_F | RX_VLAN_F | RSS_F) \
+ R(sec_vlan_ptype, R_SEC_F | RX_VLAN_F | PTYPE_F) \
+ R(sec_vlan_ptype_rss, R_SEC_F | RX_VLAN_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_cksum, R_SEC_F | RX_VLAN_F | CKSUM_F) \
+ R(sec_vlan_cksum_rss, R_SEC_F | RX_VLAN_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_cksum_ptype, R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_cksum_ptype_rss, \
+ R_SEC_F | RX_VLAN_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_mark, R_SEC_F | RX_VLAN_F | MARK_F) \
+ R(sec_vlan_mark_rss, R_SEC_F | RX_VLAN_F | MARK_F | RSS_F) \
+ R(sec_vlan_mark_ptype, R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F) \
+ R(sec_vlan_mark_ptype_rss, \
+ R_SEC_F | RX_VLAN_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_mark_cksum, R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F) \
+ R(sec_vlan_mark_cksum_rss, \
+ R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_mark_cksum_ptype, \
+ R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_mark_cksum_ptype_rss, \
+ R_SEC_F | RX_VLAN_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES_112_127 \
+ R(sec_vlan_ts, R_SEC_F | RX_VLAN_F | TS_F) \
+ R(sec_vlan_ts_rss, R_SEC_F | RX_VLAN_F | TS_F | RSS_F) \
+ R(sec_vlan_ts_ptype, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F) \
+ R(sec_vlan_ts_ptype_rss, R_SEC_F | RX_VLAN_F | TS_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_ts_cksum, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F) \
+ R(sec_vlan_ts_cksum_rss, R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_ts_cksum_ptype, \
+ R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_ts_cksum_ptype_rss, \
+ R_SEC_F | RX_VLAN_F | TS_F | CKSUM_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_ts_mark, R_SEC_F | RX_VLAN_F | TS_F | MARK_F) \
+ R(sec_vlan_ts_mark_rss, R_SEC_F | RX_VLAN_F | TS_F | MARK_F | RSS_F) \
+ R(sec_vlan_ts_mark_ptype, \
+ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F) \
+ R(sec_vlan_ts_mark_ptype_rss, \
+ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | PTYPE_F | RSS_F) \
+ R(sec_vlan_ts_mark_cksum, \
+ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F) \
+ R(sec_vlan_ts_mark_cksum_rss, \
+ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | RSS_F) \
+ R(sec_vlan_ts_mark_cksum_ptype, \
+ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F) \
+ R(sec_vlan_ts_mark_cksum_ptype_rss, \
+ R_SEC_F | RX_VLAN_F | TS_F | MARK_F | CKSUM_F | PTYPE_F | RSS_F)
+
+#define NIX_RX_FASTPATH_MODES \
+ NIX_RX_FASTPATH_MODES_0_15 \
+ NIX_RX_FASTPATH_MODES_16_31 \
+ NIX_RX_FASTPATH_MODES_32_47 \
+ NIX_RX_FASTPATH_MODES_48_63 \
+ NIX_RX_FASTPATH_MODES_64_79 \
+ NIX_RX_FASTPATH_MODES_80_95 \
+ NIX_RX_FASTPATH_MODES_96_111 \
+ NIX_RX_FASTPATH_MODES_112_127
+
+#define R(name, flags) \
uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_##name( \
void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
- \
+ \
uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_mseg_##name( \
void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
- \
+ \
uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_##name( \
void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts); \
- \
+ \
uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_mseg_##name( \
void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts);
NIX_RX_FASTPATH_MODES
#undef R
+#define NIX_RX_RECV(fn, flags) \
+ uint16_t __rte_noinline __rte_hot fn( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \
+ { \
+ return cn9k_nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags)); \
+ }
+
+#define NIX_RX_RECV_MSEG(fn, flags) NIX_RX_RECV(fn, flags | NIX_RX_MULTI_SEG_F)
+
+#define NIX_RX_RECV_VEC(fn, flags) \
+ uint16_t __rte_noinline __rte_hot fn( \
+ void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \
+ { \
+ return cn9k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts, \
+ (flags)); \
+ }
+
+#define NIX_RX_RECV_VEC_MSEG(fn, flags) \
+ NIX_RX_RECV_VEC(fn, flags | NIX_RX_MULTI_SEG_F)
+
#endif /* __CN9K_RX_H__ */
deleted file mode 100644
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_ethdev.h"
-#include "cn9k_rx.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_mseg_##name( \
- void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \
- { \
- return cn9k_nix_recv_pkts(rx_queue, rx_pkts, pkts, \
- (flags) | NIX_RX_MULTI_SEG_F); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
new file mode 100644
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include "cn9k_ethdev.h"
+#include "cn9k_rx.h"
+
+static inline void
+pick_rx_func(struct rte_eth_dev *eth_dev,
+ const eth_rx_burst_t rx_burst[NIX_RX_OFFLOAD_MAX])
+{
+ struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+
+ /* [TSP] [MARK] [VLAN] [CKSUM] [PTYPE] [RSS] */
+ eth_dev->rx_pkt_burst =
+ rx_burst[dev->rx_offload_flags & (NIX_RX_OFFLOAD_MAX - 1)];
+
+ rte_atomic_thread_fence(__ATOMIC_RELEASE);
+}
+
+void
+cn9k_eth_set_rx_function(struct rte_eth_dev *eth_dev)
+{
+ struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
+
+ const eth_rx_burst_t nix_eth_rx_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_nix_recv_pkts_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_burst_mseg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_nix_recv_pkts_mseg_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_vec_burst[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_nix_recv_pkts_vec_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ const eth_rx_burst_t nix_eth_rx_vec_burst_mseg[NIX_RX_OFFLOAD_MAX] = {
+#define R(name, flags) [flags] = cn9k_nix_recv_pkts_vec_mseg_##name,
+
+ NIX_RX_FASTPATH_MODES
+#undef R
+ };
+
+ /* Copy multi seg version with no offload for tear down sequence */
+ if (rte_eal_process_type() == RTE_PROC_PRIMARY)
+ dev->rx_pkt_burst_no_offload = nix_eth_rx_burst_mseg[0];
+
+ if (dev->scalar_ena) {
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
+ return pick_rx_func(eth_dev, nix_eth_rx_burst_mseg);
+ return pick_rx_func(eth_dev, nix_eth_rx_burst);
+ }
+
+ if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
+ return pick_rx_func(eth_dev, nix_eth_rx_vec_burst_mseg);
+ return pick_rx_func(eth_dev, nix_eth_rx_vec_burst);
+}
deleted file mode 100644
@@ -1,17 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_ethdev.h"
-#include "cn9k_rx.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_##name( \
- void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \
- { \
- return cn9k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts, \
- (flags)); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
deleted file mode 100644
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(C) 2021 Marvell.
- */
-
-#include "cn9k_ethdev.h"
-#include "cn9k_rx.h"
-
-#define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \
- uint16_t __rte_noinline __rte_hot cn9k_nix_recv_pkts_vec_mseg_##name( \
- void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t pkts) \
- { \
- return cn9k_nix_recv_pkts_vector(rx_queue, rx_pkts, pkts, \
- (flags) | \
- NIX_RX_MULTI_SEG_F); \
- }
-
-NIX_RX_FASTPATH_MODES
-#undef R
@@ -28,15 +28,48 @@ sources += files(
'cn9k_ethdev.c',
'cn9k_ethdev_sec.c',
'cn9k_rte_flow.c',
- 'cn9k_rx.c',
- 'cn9k_rx_mseg.c',
- 'cn9k_rx_vec.c',
- 'cn9k_rx_vec_mseg.c',
+ 'cn9k_rx_select.c',
'cn9k_tx.c',
'cn9k_tx_mseg.c',
'cn9k_tx_vec.c',
'cn9k_tx_vec_mseg.c',
)
+
+sources += files(
+ 'rx/cn9k/rx_0_15.c',
+ 'rx/cn9k/rx_16_31.c',
+ 'rx/cn9k/rx_32_47.c',
+ 'rx/cn9k/rx_48_63.c',
+ 'rx/cn9k/rx_64_79.c',
+ 'rx/cn9k/rx_80_95.c',
+ 'rx/cn9k/rx_96_111.c',
+ 'rx/cn9k/rx_112_127.c',
+ 'rx/cn9k/rx_0_15_mseg.c',
+ 'rx/cn9k/rx_16_31_mseg.c',
+ 'rx/cn9k/rx_32_47_mseg.c',
+ 'rx/cn9k/rx_48_63_mseg.c',
+ 'rx/cn9k/rx_64_79_mseg.c',
+ 'rx/cn9k/rx_80_95_mseg.c',
+ 'rx/cn9k/rx_96_111_mseg.c',
+ 'rx/cn9k/rx_112_127_mseg.c',
+ 'rx/cn9k/rx_0_15_vec.c',
+ 'rx/cn9k/rx_16_31_vec.c',
+ 'rx/cn9k/rx_32_47_vec.c',
+ 'rx/cn9k/rx_48_63_vec.c',
+ 'rx/cn9k/rx_64_79_vec.c',
+ 'rx/cn9k/rx_80_95_vec.c',
+ 'rx/cn9k/rx_96_111_vec.c',
+ 'rx/cn9k/rx_112_127_vec.c',
+ 'rx/cn9k/rx_0_15_vec_mseg.c',
+ 'rx/cn9k/rx_16_31_vec_mseg.c',
+ 'rx/cn9k/rx_32_47_vec_mseg.c',
+ 'rx/cn9k/rx_48_63_vec_mseg.c',
+ 'rx/cn9k/rx_64_79_vec_mseg.c',
+ 'rx/cn9k/rx_80_95_vec_mseg.c',
+ 'rx/cn9k/rx_96_111_vec_mseg.c',
+ 'rx/cn9k/rx_112_127_vec_mseg.c',
+)
+
# CN10K
sources += files(
'cn10k_ethdev.c',