@@ -108,6 +108,10 @@ New Features
User can choose length or protocol header to configure buffer split
according to NIC's capability.
+* **Updated Intel ice driver.**
+
+ * Added protocol based buffer split support in scalar path.
+
Removed Items
-------------
@@ -159,6 +159,7 @@ static int ice_timesync_read_time(struct rte_eth_dev *dev,
static int ice_timesync_write_time(struct rte_eth_dev *dev,
const struct timespec *timestamp);
static int ice_timesync_disable(struct rte_eth_dev *dev);
+static const uint32_t *ice_buffer_split_supported_hdr_ptypes_get(struct rte_eth_dev *dev);
static const struct rte_pci_id pci_id_ice_map[] = {
{ RTE_PCI_DEVICE(ICE_INTEL_VENDOR_ID, ICE_DEV_ID_E823L_BACKPLANE) },
@@ -270,6 +271,7 @@ static const struct eth_dev_ops ice_eth_dev_ops = {
.timesync_write_time = ice_timesync_write_time,
.timesync_disable = ice_timesync_disable,
.tm_ops_get = ice_tm_ops_get,
+ .buffer_split_supported_hdr_ptypes_get = ice_buffer_split_supported_hdr_ptypes_get,
};
/* store statistics names and its offset in stats structure */
@@ -3797,7 +3799,8 @@ ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM |
RTE_ETH_RX_OFFLOAD_VLAN_EXTEND |
RTE_ETH_RX_OFFLOAD_RSS_HASH |
- RTE_ETH_RX_OFFLOAD_TIMESTAMP;
+ RTE_ETH_RX_OFFLOAD_TIMESTAMP |
+ RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT;
dev_info->tx_offload_capa |=
RTE_ETH_TX_OFFLOAD_QINQ_INSERT |
RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
@@ -3809,7 +3812,7 @@ ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
dev_info->flow_type_rss_offloads |= ICE_RSS_OFFLOAD_ALL;
}
- dev_info->rx_queue_offload_capa = 0;
+ dev_info->rx_queue_offload_capa = RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT;
dev_info->tx_queue_offload_capa = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
dev_info->reta_size = pf->hash_lut_size;
@@ -3878,6 +3881,11 @@ ice_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
dev_info->default_rxportconf.ring_size = ICE_BUF_SIZE_MIN;
dev_info->default_txportconf.ring_size = ICE_BUF_SIZE_MIN;
+ dev_info->rx_seg_capa.max_nseg = ICE_RX_MAX_NSEG;
+ dev_info->rx_seg_capa.multi_pools = 1;
+ dev_info->rx_seg_capa.offset_allowed = 0;
+ dev_info->rx_seg_capa.offset_align_log2 = 0;
+
return 0;
}
@@ -5932,6 +5940,49 @@ ice_timesync_disable(struct rte_eth_dev *dev)
return 0;
}
+static const uint32_t *
+ice_buffer_split_supported_hdr_ptypes_get(struct rte_eth_dev *dev __rte_unused)
+{
+ /* Buffer split protocol header capability. */
+ static const uint32_t ptypes[] = {
+ /* Non tunneled */
+ RTE_PTYPE_L2_ETHER,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_SCTP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
+ RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_SCTP,
+
+ /* Tunneled */
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP,
+
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_UDP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP,
+ RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
+ RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_SCTP,
+
+ RTE_PTYPE_UNKNOWN
+ };
+
+ return ptypes;
+}
+
static int
ice_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct rte_pci_device *pci_dev)
@@ -259,7 +259,6 @@ ice_program_hw_rx_queue(struct ice_rx_queue *rxq)
/* Set buffer size as the head split is disabled. */
buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
RTE_PKTMBUF_HEADROOM);
- rxq->rx_hdr_len = 0;
rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << ICE_RLAN_CTX_DBUF_S));
rxq->max_pkt_len =
RTE_MIN((uint32_t)ICE_SUPPORT_CHAIN_NUM * rxq->rx_buf_len,
@@ -288,11 +287,84 @@ ice_program_hw_rx_queue(struct ice_rx_queue *rxq)
memset(&rx_ctx, 0, sizeof(rx_ctx));
+ if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) {
+ uint32_t proto_hdr;
+ proto_hdr = rxq->rxseg[0].proto_hdr;
+
+ if (proto_hdr == RTE_PTYPE_UNKNOWN) {
+ PMD_DRV_LOG(ERR, "Buffer split protocol must be configured");
+ return -EINVAL;
+ }
+
+ switch (proto_hdr & RTE_PTYPE_L4_MASK) {
+ case RTE_PTYPE_L4_TCP:
+ case RTE_PTYPE_L4_UDP:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_TCP_UDP;
+ goto set_hsplit_finish;
+ case RTE_PTYPE_L4_SCTP:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_SCTP;
+ goto set_hsplit_finish;
+ }
+
+ switch (proto_hdr & RTE_PTYPE_L3_MASK) {
+ case RTE_PTYPE_L3_IPV4_EXT_UNKNOWN:
+ case RTE_PTYPE_L3_IPV6_EXT_UNKNOWN:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_IP;
+ goto set_hsplit_finish;
+ }
+
+ switch (proto_hdr & RTE_PTYPE_L2_MASK) {
+ case RTE_PTYPE_L2_ETHER:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_L2;
+ rx_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_SPLIT_L2;
+ goto set_hsplit_finish;
+ }
+
+ switch (proto_hdr & RTE_PTYPE_INNER_L4_MASK) {
+ case RTE_PTYPE_INNER_L4_TCP:
+ case RTE_PTYPE_INNER_L4_UDP:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_TCP_UDP;
+ goto set_hsplit_finish;
+ case RTE_PTYPE_INNER_L4_SCTP:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_SCTP;
+ goto set_hsplit_finish;
+ }
+
+ switch (proto_hdr & RTE_PTYPE_INNER_L3_MASK) {
+ case RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN:
+ case RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_IP;
+ goto set_hsplit_finish;
+ }
+
+ switch (proto_hdr & RTE_PTYPE_INNER_L2_MASK) {
+ case RTE_PTYPE_INNER_L2_ETHER:
+ rx_ctx.dtype = ICE_RX_DTYPE_HEADER_SPLIT;
+ rx_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_SPLIT_L2;
+ goto set_hsplit_finish;
+ }
+
+ PMD_DRV_LOG(ERR, "Buffer split protocol is not supported");
+ return -EINVAL;
+
+set_hsplit_finish:
+ rxq->rx_hdr_len = ICE_RX_HDR_BUF_SIZE;
+ } else {
+ rxq->rx_hdr_len = 0;
+ rx_ctx.dtype = 0; /* No Protocol Based Buffer Split mode */
+ }
+
rx_ctx.base = rxq->rx_ring_dma / ICE_QUEUE_BASE_ADDR_UNIT;
rx_ctx.qlen = rxq->nb_rx_desc;
rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S;
- rx_ctx.dtype = 0; /* No Header Split mode */
#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
rx_ctx.dsize = 1; /* 32B descriptors */
#endif
@@ -378,6 +450,7 @@ ice_alloc_rx_queue_mbufs(struct ice_rx_queue *rxq)
for (i = 0; i < rxq->nb_rx_desc; i++) {
volatile union ice_rx_flex_desc *rxd;
+ rxd = &rxq->rx_ring[i];
struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp);
if (unlikely(!mbuf)) {
@@ -385,8 +458,6 @@ ice_alloc_rx_queue_mbufs(struct ice_rx_queue *rxq)
return -ENOMEM;
}
- rte_mbuf_refcnt_set(mbuf, 1);
- mbuf->next = NULL;
mbuf->data_off = RTE_PKTMBUF_HEADROOM;
mbuf->nb_segs = 1;
mbuf->port = rxq->port_id;
@@ -394,9 +465,32 @@ ice_alloc_rx_queue_mbufs(struct ice_rx_queue *rxq)
dma_addr =
rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf));
- rxd = &rxq->rx_ring[i];
- rxd->read.pkt_addr = dma_addr;
- rxd->read.hdr_addr = 0;
+ if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ rte_mbuf_refcnt_set(mbuf, 1);
+ mbuf->next = NULL;
+ rxd->read.hdr_addr = 0;
+ rxd->read.pkt_addr = dma_addr;
+ } else {
+ struct rte_mbuf *mbuf_pay;
+ mbuf_pay = rte_mbuf_raw_alloc(rxq->rxseg[1].mp);
+ if (unlikely(!mbuf_pay)) {
+ PMD_DRV_LOG(ERR, "Failed to allocate payload mbuf for RX");
+ return -ENOMEM;
+ }
+
+ mbuf_pay->next = NULL;
+ mbuf_pay->data_off = RTE_PKTMBUF_HEADROOM;
+ mbuf_pay->nb_segs = 1;
+ mbuf_pay->port = rxq->port_id;
+ mbuf->next = mbuf_pay;
+
+ rxd->read.hdr_addr = dma_addr;
+ /* The LS bit should be set to zero regardless of
+ * buffer split enablement.
+ */
+ rxd->read.pkt_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbuf_pay));
+ }
+
#ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC
rxd->read.rsvd1 = 0;
rxd->read.rsvd2 = 0;
@@ -420,14 +514,14 @@ _ice_rx_queue_release_mbufs(struct ice_rx_queue *rxq)
for (i = 0; i < rxq->nb_rx_desc; i++) {
if (rxq->sw_ring[i].mbuf) {
- rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
+ rte_pktmbuf_free(rxq->sw_ring[i].mbuf);
rxq->sw_ring[i].mbuf = NULL;
}
}
if (rxq->rx_nb_avail == 0)
return;
for (i = 0; i < rxq->rx_nb_avail; i++)
- rte_pktmbuf_free_seg(rxq->rx_stage[rxq->rx_next_avail + i]);
+ rte_pktmbuf_free(rxq->rx_stage[rxq->rx_next_avail + i]);
rxq->rx_nb_avail = 0;
}
@@ -719,7 +813,7 @@ ice_fdir_program_hw_rx_queue(struct ice_rx_queue *rxq)
rx_ctx.qlen = rxq->nb_rx_desc;
rx_ctx.dbuf = rxq->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
rx_ctx.hbuf = rxq->rx_hdr_len >> ICE_RLAN_CTX_HBUF_S;
- rx_ctx.dtype = 0; /* No Header Split mode */
+ rx_ctx.dtype = 0; /* No Buffer Split mode */
rx_ctx.dsize = 1; /* 32B descriptors */
rx_ctx.rxmax = ICE_ETH_MAX_LEN;
/* TPH: Transaction Layer Packet (TLP) processing hints */
@@ -1053,6 +1147,8 @@ ice_rx_queue_setup(struct rte_eth_dev *dev,
uint16_t len;
int use_def_burst_func = 1;
uint64_t offloads;
+ uint16_t n_seg = rx_conf->rx_nseg;
+ const struct rte_eth_rxseg_split *rx_seg;
if (nb_desc % ICE_ALIGN_RING_DESC != 0 ||
nb_desc > ICE_MAX_RING_DESC ||
@@ -1064,6 +1160,15 @@ ice_rx_queue_setup(struct rte_eth_dev *dev,
offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
+ if (mp)
+ n_seg = 1;
+
+ if (n_seg > 1 && !(offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ PMD_INIT_LOG(ERR, "port %u queue index %u split offload not configured",
+ dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+
/* Free memory if needed */
if (dev->data->rx_queues[queue_idx]) {
ice_rx_queue_release(dev->data->rx_queues[queue_idx]);
@@ -1075,12 +1180,22 @@ ice_rx_queue_setup(struct rte_eth_dev *dev,
sizeof(struct ice_rx_queue),
RTE_CACHE_LINE_SIZE,
socket_id);
+
if (!rxq) {
PMD_INIT_LOG(ERR, "Failed to allocate memory for "
"rx queue data structure");
return -ENOMEM;
}
- rxq->mp = mp;
+
+ rxq->rxseg_nb = n_seg;
+ if (n_seg > 1) {
+ rx_seg = (const struct rte_eth_rxseg_split *)rx_conf->rx_seg;
+ rte_memcpy(rxq->rxseg, rx_seg, sizeof(struct rte_eth_rxseg_split) * n_seg);
+ rxq->mp = rxq->rxseg[0].mp;
+ } else {
+ rxq->mp = mp;
+ }
+
rxq->nb_rx_desc = nb_desc;
rxq->rx_free_thresh = rx_conf->rx_free_thresh;
rxq->queue_id = queue_idx;
@@ -1551,7 +1666,7 @@ ice_rx_scan_hw_ring(struct ice_rx_queue *rxq)
struct ice_rx_entry *rxep;
struct rte_mbuf *mb;
uint16_t stat_err0;
- uint16_t pkt_len;
+ uint16_t pkt_len, hdr_len;
int32_t s[ICE_LOOK_AHEAD], nb_dd;
int32_t i, j, nb_rx = 0;
uint64_t pkt_flags = 0;
@@ -1606,6 +1721,27 @@ ice_rx_scan_hw_ring(struct ice_rx_queue *rxq)
ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
mb->data_len = pkt_len;
mb->pkt_len = pkt_len;
+
+ if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) &
+ ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
+ mb->data_len = pkt_len;
+ mb->pkt_len = pkt_len;
+ } else {
+ mb->nb_segs = (uint16_t)(mb->nb_segs + mb->next->nb_segs);
+ mb->next->next = NULL;
+ hdr_len = rte_le_to_cpu_16(rxdp[j].wb.hdr_len_sph_flex_flags1) &
+ ICE_RX_FLEX_DESC_HEADER_LEN_M;
+ pkt_len = (rte_le_to_cpu_16(rxdp[j].wb.pkt_len) &
+ ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
+ mb->data_len = hdr_len;
+ mb->pkt_len = hdr_len + pkt_len;
+ mb->next->data_len = pkt_len;
+#ifdef RTE_ETHDEV_DEBUG_RX
+ rte_pktmbuf_dump(stdout, mb, rte_pktmbuf_pkt_len(mb));
+#endif
+ }
+
mb->ol_flags = 0;
stat_err0 = rte_le_to_cpu_16(rxdp[j].wb.status_error0);
pkt_flags = ice_rxd_error_to_pkt_flags(stat_err0);
@@ -1697,7 +1833,9 @@ ice_rx_alloc_bufs(struct ice_rx_queue *rxq)
struct rte_mbuf *mb;
uint16_t alloc_idx, i;
uint64_t dma_addr;
- int diag;
+ int diag, diag_pay;
+ uint64_t pay_addr;
+ struct rte_mbuf *mbufs_pay[rxq->rx_free_thresh];
/* Allocate buffers in bulk */
alloc_idx = (uint16_t)(rxq->rx_free_trigger -
@@ -1710,6 +1848,15 @@ ice_rx_alloc_bufs(struct ice_rx_queue *rxq)
return -ENOMEM;
}
+ if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) {
+ diag_pay = rte_mempool_get_bulk(rxq->rxseg[1].mp,
+ (void *)mbufs_pay, rxq->rx_free_thresh);
+ if (unlikely(diag_pay != 0)) {
+ PMD_RX_LOG(ERR, "Failed to get payload mbufs in bulk");
+ return -ENOMEM;
+ }
+ }
+
rxdp = &rxq->rx_ring[alloc_idx];
for (i = 0; i < rxq->rx_free_thresh; i++) {
if (likely(i < (rxq->rx_free_thresh - 1)))
@@ -1718,13 +1865,21 @@ ice_rx_alloc_bufs(struct ice_rx_queue *rxq)
mb = rxep[i].mbuf;
rte_mbuf_refcnt_set(mb, 1);
- mb->next = NULL;
mb->data_off = RTE_PKTMBUF_HEADROOM;
mb->nb_segs = 1;
mb->port = rxq->port_id;
dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mb));
- rxdp[i].read.hdr_addr = 0;
- rxdp[i].read.pkt_addr = dma_addr;
+
+ if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ mb->next = NULL;
+ rxdp[i].read.hdr_addr = 0;
+ rxdp[i].read.pkt_addr = dma_addr;
+ } else {
+ mb->next = mbufs_pay[i];
+ pay_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(mbufs_pay[i]));
+ rxdp[i].read.hdr_addr = dma_addr;
+ rxdp[i].read.pkt_addr = pay_addr;
+ }
}
/* Update Rx tail register */
@@ -2333,11 +2488,13 @@ ice_recv_pkts(void *rx_queue,
struct ice_rx_entry *sw_ring = rxq->sw_ring;
struct ice_rx_entry *rxe;
struct rte_mbuf *nmb; /* new allocated mbuf */
+ struct rte_mbuf *nmb_pay; /* new allocated payload mbuf */
struct rte_mbuf *rxm; /* pointer to store old mbuf in SW ring */
uint16_t rx_id = rxq->rx_tail;
uint16_t nb_rx = 0;
uint16_t nb_hold = 0;
uint16_t rx_packet_len;
+ uint16_t rx_header_len;
uint16_t rx_stat_err0;
uint64_t dma_addr;
uint64_t pkt_flags;
@@ -2365,12 +2522,16 @@ ice_recv_pkts(void *rx_queue,
if (!(rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_DD_S)))
break;
- /* allocate mbuf */
+ if (rx_stat_err0 & (1 << ICE_RX_FLEX_DESC_STATUS0_HBO_S))
+ break;
+
+ /* allocate header mbuf */
nmb = rte_mbuf_raw_alloc(rxq->mp);
if (unlikely(!nmb)) {
rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++;
break;
}
+
rxd = *rxdp; /* copy descriptor in ring to temp variable*/
nb_hold++;
@@ -2383,24 +2544,60 @@ ice_recv_pkts(void *rx_queue,
dma_addr =
rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb));
- /**
- * fill the read format of descriptor with physic address in
- * new allocated mbuf: nmb
- */
- rxdp->read.hdr_addr = 0;
- rxdp->read.pkt_addr = dma_addr;
+ if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ /**
+ * fill the read format of descriptor with physic address in
+ * new allocated mbuf: nmb
+ */
+ rxdp->read.hdr_addr = 0;
+ rxdp->read.pkt_addr = dma_addr;
+ } else {
+ /* allocate payload mbuf */
+ nmb_pay = rte_mbuf_raw_alloc(rxq->rxseg[1].mp);
+ if (unlikely(!nmb_pay)) {
+ rxq->vsi->adapter->pf.dev_data->rx_mbuf_alloc_failed++;
+ break;
+ }
+
+ nmb->next = nmb_pay;
+ nmb_pay->next = NULL;
- /* calculate rx_packet_len of the received pkt */
- rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) &
- ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
+ /**
+ * fill the read format of descriptor with physic address in
+ * new allocated mbuf: nmb
+ */
+ rxdp->read.hdr_addr = dma_addr;
+ rxdp->read.pkt_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(nmb_pay));
+ }
/* fill old mbuf with received descriptor: rxd */
rxm->data_off = RTE_PKTMBUF_HEADROOM;
rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
- rxm->nb_segs = 1;
- rxm->next = NULL;
- rxm->pkt_len = rx_packet_len;
- rxm->data_len = rx_packet_len;
+ if (!(rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)) {
+ rxm->nb_segs = 1;
+ rxm->next = NULL;
+ /* calculate rx_packet_len of the received pkt */
+ rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) &
+ ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
+ rxm->data_len = rx_packet_len;
+ rxm->pkt_len = rx_packet_len;
+ } else {
+ rxm->nb_segs = (uint16_t)(rxm->nb_segs + rxm->next->nb_segs);
+ rxm->next->next = NULL;
+ /* calculate rx_packet_len of the received pkt */
+ rx_header_len = rte_le_to_cpu_16(rxd.wb.hdr_len_sph_flex_flags1) &
+ ICE_RX_FLEX_DESC_HEADER_LEN_M;
+ rx_packet_len = (rte_le_to_cpu_16(rxd.wb.pkt_len) &
+ ICE_RX_FLX_DESC_PKT_LEN_M) - rxq->crc_len;
+ rxm->data_len = rx_header_len;
+ rxm->pkt_len = rx_header_len + rx_packet_len;
+ rxm->next->data_len = rx_packet_len;
+
+#ifdef RTE_ETHDEV_DEBUG_RX
+ rte_pktmbuf_dump(stdout, rxm, rte_pktmbuf_pkt_len(rxm));
+#endif
+ }
+
rxm->port = rxq->port_id;
rxm->packet_type = ptype_tbl[ICE_RX_FLEX_DESC_PTYPE_M &
rte_le_to_cpu_16(rxd.wb.ptype_flex_flags0)];
@@ -16,6 +16,9 @@
#define ICE_RX_MAX_BURST 32
#define ICE_TX_MAX_BURST 32
+/* Maximal number of segments to split. */
+#define ICE_RX_MAX_NSEG 2
+
#define ICE_CHK_Q_ENA_COUNT 100
#define ICE_CHK_Q_ENA_INTERVAL_US 100
@@ -43,6 +46,11 @@
extern uint64_t ice_timestamp_dynflag;
extern int ice_timestamp_dynfield_offset;
+/* Max header size can be 2K - 64 bytes */
+#define ICE_RX_HDR_BUF_SIZE (2048 - 64)
+
+#define ICE_HEADER_SPLIT_ENA BIT(0)
+
typedef void (*ice_rx_release_mbufs_t)(struct ice_rx_queue *rxq);
typedef void (*ice_tx_release_mbufs_t)(struct ice_tx_queue *txq);
typedef void (*ice_rxd_to_pkt_fields_t)(struct ice_rx_queue *rxq,
@@ -53,6 +61,12 @@ struct ice_rx_entry {
struct rte_mbuf *mbuf;
};
+enum ice_rx_dtype {
+ ICE_RX_DTYPE_NO_SPLIT = 0,
+ ICE_RX_DTYPE_HEADER_SPLIT = 1,
+ ICE_RX_DTYPE_SPLIT_ALWAYS = 2,
+};
+
struct ice_rx_queue {
struct rte_mempool *mp; /* mbuf pool to populate RX ring */
volatile union ice_rx_flex_desc *rx_ring;/* RX ring virtual address */
@@ -99,6 +113,8 @@ struct ice_rx_queue {
uint32_t hw_time_high; /* high 32 bits of timestamp */
uint32_t hw_time_low; /* low 32 bits of timestamp */
uint64_t hw_time_update; /* SW time of HW record updating */
+ struct rte_eth_rxseg_split rxseg[ICE_RX_MAX_NSEG];
+ uint32_t rxseg_nb;
};
struct ice_tx_entry {
@@ -291,6 +291,9 @@ ice_rx_vec_queue_default(struct ice_rx_queue *rxq)
if (rxq->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)
return -1;
+ if (rxq->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT)
+ return -1;
+
if (rxq->offloads & ICE_RX_VECTOR_OFFLOAD)
return ICE_VECTOR_OFFLOAD_PATH;