@@ -25,6 +25,7 @@
#define GVE_DEFAULT_RX_FREE_THRESH 512
#define GVE_DEFAULT_TX_FREE_THRESH 256
+#define GVE_TX_MAX_FREE_SZ 512
/* PTYPEs are always 10 bits. */
#define GVE_NUM_PTYPES 1024
@@ -45,6 +46,18 @@ union gve_tx_desc {
struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
};
+/* Offload features */
+union gve_tx_offload {
+ uint64_t data;
+ struct {
+ uint64_t l2_len:7; /* L2 (MAC) Header Length. */
+ uint64_t l3_len:9; /* L3 (IP) Header Length. */
+ uint64_t l4_len:8; /* L4 Header Length. */
+ uint64_t tso_segsz:16; /* TCP TSO segment size */
+ /* uint64_t unused : 24; */
+ };
+};
+
struct gve_tx_iovec {
uint32_t iov_base; /* offset in fifo */
uint32_t iov_len;
@@ -298,4 +311,8 @@ void gve_stop_tx_queues(struct rte_eth_dev *dev);
void gve_stop_rx_queues(struct rte_eth_dev *dev);
+uint16_t gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+
+uint16_t gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+
#endif /* _GVE_H_ */
@@ -583,6 +583,11 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
if (err)
return err;
+ if (gve_is_gqi(priv)) {
+ eth_dev->rx_pkt_burst = gve_rx_burst;
+ eth_dev->tx_pkt_burst = gve_tx_burst;
+ }
+
eth_dev->data->mac_addrs = rte_zmalloc("gve_mac", sizeof(struct rte_ether_addr), 0);
if (!eth_dev->data->mac_addrs) {
PMD_INIT_LOG(ERR, "Failed to allocate memory to store mac address");
@@ -5,6 +5,149 @@
#include "gve.h"
#include "gve_adminq.h"
+static inline void
+gve_rx_refill(struct gve_rx_queue *rxq)
+{
+ uint16_t mask = rxq->nb_rx_desc - 1;
+ uint16_t idx = rxq->next_avail & mask;
+ uint32_t next_avail = rxq->next_avail;
+ uint16_t nb_alloc, i;
+ struct rte_mbuf *nmb;
+ int diag;
+
+ /* wrap around */
+ nb_alloc = rxq->nb_rx_desc - idx;
+ if (nb_alloc <= rxq->nb_avail) {
+ diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);
+ if (diag < 0) {
+ for (i = 0; i < nb_alloc; i++) {
+ nmb = rte_pktmbuf_alloc(rxq->mpool);
+ if (!nmb)
+ break;
+ rxq->sw_ring[idx + i] = nmb;
+ }
+ if (i != nb_alloc)
+ nb_alloc = i;
+ }
+ rxq->nb_avail -= nb_alloc;
+ next_avail += nb_alloc;
+
+ /* queue page list mode doesn't need real refill. */
+ if (rxq->is_gqi_qpl) {
+ idx += nb_alloc;
+ } else {
+ for (i = 0; i < nb_alloc; i++) {
+ nmb = rxq->sw_ring[idx];
+ rxq->rx_data_ring[idx].addr =
+ rte_cpu_to_be_64(rte_mbuf_data_iova(nmb));
+ idx++;
+ }
+ }
+ if (idx == rxq->nb_rx_desc)
+ idx = 0;
+ }
+
+ if (rxq->nb_avail > 0) {
+ nb_alloc = rxq->nb_avail;
+ if (rxq->nb_rx_desc < idx + rxq->nb_avail)
+ nb_alloc = rxq->nb_rx_desc - idx;
+ diag = rte_pktmbuf_alloc_bulk(rxq->mpool, &rxq->sw_ring[idx], nb_alloc);
+ if (diag < 0) {
+ for (i = 0; i < nb_alloc; i++) {
+ nmb = rte_pktmbuf_alloc(rxq->mpool);
+ if (!nmb)
+ break;
+ rxq->sw_ring[idx + i] = nmb;
+ }
+ nb_alloc = i;
+ }
+ rxq->nb_avail -= nb_alloc;
+ next_avail += nb_alloc;
+
+ if (!rxq->is_gqi_qpl) {
+ for (i = 0; i < nb_alloc; i++) {
+ nmb = rxq->sw_ring[idx];
+ rxq->rx_data_ring[idx].addr =
+ rte_cpu_to_be_64(rte_mbuf_data_iova(nmb));
+ idx++;
+ }
+ }
+ }
+
+ if (next_avail != rxq->next_avail) {
+ rte_write32(rte_cpu_to_be_32(next_avail), rxq->qrx_tail);
+ rxq->next_avail = next_avail;
+ }
+}
+
+uint16_t
+gve_rx_burst(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ volatile struct gve_rx_desc *rxr, *rxd;
+ struct gve_rx_queue *rxq = rx_queue;
+ uint16_t rx_id = rxq->rx_tail;
+ struct rte_mbuf *rxe;
+ uint16_t nb_rx, len;
+ uint64_t addr;
+
+ rxr = rxq->rx_desc_ring;
+
+ for (nb_rx = 0; nb_rx < nb_pkts; nb_rx++) {
+ rxd = &rxr[rx_id];
+ if (GVE_SEQNO(rxd->flags_seq) != rxq->expected_seqno)
+ break;
+
+ if (rxd->flags_seq & GVE_RXF_ERR)
+ continue;
+
+ len = rte_be_to_cpu_16(rxd->len) - GVE_RX_PAD;
+ rxe = rxq->sw_ring[rx_id];
+ rxe->data_off = RTE_PKTMBUF_HEADROOM;
+ if (rxq->is_gqi_qpl) {
+ addr = (uint64_t)(rxq->qpl->mz->addr) + rx_id * PAGE_SIZE + GVE_RX_PAD;
+ rte_memcpy((void *)((uint64_t)rxe->buf_addr + rxe->data_off),
+ (void *)addr, len);
+ }
+ rxe->nb_segs = 1;
+ rxe->next = NULL;
+ rxe->pkt_len = len;
+ rxe->data_len = len;
+ rxe->port = rxq->port_id;
+ rxe->packet_type = 0;
+ rxe->ol_flags = 0;
+
+ if (rxd->flags_seq & GVE_RXF_TCP)
+ rxe->packet_type |= RTE_PTYPE_L4_TCP;
+ if (rxd->flags_seq & GVE_RXF_UDP)
+ rxe->packet_type |= RTE_PTYPE_L4_UDP;
+ if (rxd->flags_seq & GVE_RXF_IPV4)
+ rxe->packet_type |= RTE_PTYPE_L3_IPV4;
+ if (rxd->flags_seq & GVE_RXF_IPV6)
+ rxe->packet_type |= RTE_PTYPE_L3_IPV6;
+
+ if (gve_needs_rss(rxd->flags_seq)) {
+ rxe->ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
+ rxe->hash.rss = rte_be_to_cpu_32(rxd->rss_hash);
+ }
+
+ rxq->expected_seqno = gve_next_seqno(rxq->expected_seqno);
+
+ rx_id++;
+ if (rx_id == rxq->nb_rx_desc)
+ rx_id = 0;
+
+ rx_pkts[nb_rx] = rxe;
+ }
+
+ rxq->nb_avail += nb_rx;
+ rxq->rx_tail = rx_id;
+
+ if (rxq->nb_avail > rxq->free_thresh)
+ gve_rx_refill(rxq);
+
+ return nb_rx;
+}
+
static inline void
gve_reset_rxq(struct gve_rx_queue *rxq)
{
@@ -5,6 +5,458 @@
#include "gve.h"
#include "gve_adminq.h"
+static inline void
+gve_free_bulk_mbuf(struct rte_mbuf **txep, int num)
+{
+ struct rte_mbuf *m, *free[GVE_TX_MAX_FREE_SZ];
+ int nb_free = 0;
+ int i, s;
+
+ if (unlikely(num == 0))
+ return;
+
+ /* Find the 1st mbuf which needs to be free */
+ for (s = 0; s < num; s++) {
+ if (txep[s] != NULL) {
+ m = rte_pktmbuf_prefree_seg(txep[s]);
+ if (m != NULL)
+ break;
+ }
+ }
+
+ if (s == num)
+ return;
+
+ free[0] = m;
+ nb_free = 1;
+ for (i = s + 1; i < num; i++) {
+ if (likely(txep[i] != NULL)) {
+ m = rte_pktmbuf_prefree_seg(txep[i]);
+ if (likely(m != NULL)) {
+ if (likely(m->pool == free[0]->pool)) {
+ free[nb_free++] = m;
+ } else {
+ rte_mempool_put_bulk(free[0]->pool, (void *)free, nb_free);
+ free[0] = m;
+ nb_free = 1;
+ }
+ }
+ txep[i] = NULL;
+ }
+ }
+ rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
+}
+
+static inline void
+gve_tx_clean(struct gve_tx_queue *txq)
+{
+ uint16_t mask = txq->nb_tx_desc - 1;
+ uint32_t start = txq->next_to_clean & mask;
+ uint32_t ntc, nb_clean, i;
+ struct gve_tx_iovec *iov;
+
+ ntc = rte_be_to_cpu_32(rte_read32(txq->qtx_head));
+ ntc = ntc & mask;
+
+ if (ntc == start)
+ return;
+
+ /* if wrap around, free twice. */
+ if (ntc < start) {
+ nb_clean = txq->nb_tx_desc - start;
+ if (nb_clean > GVE_TX_MAX_FREE_SZ)
+ nb_clean = GVE_TX_MAX_FREE_SZ;
+ if (txq->is_gqi_qpl) {
+ for (i = start; i < start + nb_clean; i++) {
+ iov = &txq->iov_ring[i];
+ txq->fifo_avail += iov->iov_len;
+ iov->iov_base = 0;
+ iov->iov_len = 0;
+ }
+ } else {
+ gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);
+ }
+ txq->nb_free += nb_clean;
+ start += nb_clean;
+ if (start == txq->nb_tx_desc)
+ start = 0;
+ txq->next_to_clean += nb_clean;
+ }
+
+ if (ntc > start) {
+ nb_clean = ntc - start;
+ if (nb_clean > GVE_TX_MAX_FREE_SZ)
+ nb_clean = GVE_TX_MAX_FREE_SZ;
+ if (txq->is_gqi_qpl) {
+ for (i = start; i < start + nb_clean; i++) {
+ iov = &txq->iov_ring[i];
+ txq->fifo_avail += iov->iov_len;
+ iov->iov_base = 0;
+ iov->iov_len = 0;
+ }
+ } else {
+ gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);
+ }
+ txq->nb_free += nb_clean;
+ txq->next_to_clean += nb_clean;
+ }
+}
+
+static inline void
+gve_tx_clean_swr_qpl(struct gve_tx_queue *txq)
+{
+ uint32_t start = txq->sw_ntc;
+ uint32_t ntc, nb_clean;
+
+ ntc = txq->sw_tail;
+
+ if (ntc == start)
+ return;
+
+ /* if wrap around, free twice. */
+ if (ntc < start) {
+ nb_clean = txq->nb_tx_desc - start;
+ if (nb_clean > GVE_TX_MAX_FREE_SZ)
+ nb_clean = GVE_TX_MAX_FREE_SZ;
+ gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);
+
+ txq->sw_nb_free += nb_clean;
+ start += nb_clean;
+ if (start == txq->nb_tx_desc)
+ start = 0;
+ txq->sw_ntc = start;
+ }
+
+ if (ntc > start) {
+ nb_clean = ntc - start;
+ if (nb_clean > GVE_TX_MAX_FREE_SZ)
+ nb_clean = GVE_TX_MAX_FREE_SZ;
+ gve_free_bulk_mbuf(&txq->sw_ring[start], nb_clean);
+ txq->sw_nb_free += nb_clean;
+ start += nb_clean;
+ txq->sw_ntc = start;
+ }
+}
+
+static inline void
+gve_tx_fill_pkt_desc(volatile union gve_tx_desc *desc, struct rte_mbuf *mbuf,
+ uint8_t desc_cnt, uint16_t len, uint64_t addr)
+{
+ uint64_t csum_l4 = mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK;
+ uint8_t l4_csum_offset = 0;
+ uint8_t l4_hdr_offset = 0;
+
+ if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
+ csum_l4 |= RTE_MBUF_F_TX_TCP_CKSUM;
+
+ switch (csum_l4) {
+ case RTE_MBUF_F_TX_TCP_CKSUM:
+ l4_csum_offset = offsetof(struct rte_tcp_hdr, cksum);
+ l4_hdr_offset = mbuf->l2_len + mbuf->l3_len;
+ break;
+ case RTE_MBUF_F_TX_UDP_CKSUM:
+ l4_csum_offset = offsetof(struct rte_udp_hdr, dgram_cksum);
+ l4_hdr_offset = mbuf->l2_len + mbuf->l3_len;
+ break;
+ case RTE_MBUF_F_TX_SCTP_CKSUM:
+ l4_csum_offset = offsetof(struct rte_sctp_hdr, cksum);
+ l4_hdr_offset = mbuf->l2_len + mbuf->l3_len;
+ break;
+ }
+
+ if (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+ desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM;
+ desc->pkt.l4_csum_offset = l4_csum_offset >> 1;
+ desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
+ } else if (mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) {
+ desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM;
+ desc->pkt.l4_csum_offset = l4_csum_offset >> 1;
+ desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1;
+ } else {
+ desc->pkt.type_flags = GVE_TXD_STD;
+ desc->pkt.l4_csum_offset = 0;
+ desc->pkt.l4_hdr_offset = 0;
+ }
+ desc->pkt.desc_cnt = desc_cnt;
+ desc->pkt.len = rte_cpu_to_be_16(mbuf->pkt_len);
+ desc->pkt.seg_len = rte_cpu_to_be_16(len);
+ desc->pkt.seg_addr = rte_cpu_to_be_64(addr);
+}
+
+static inline void
+gve_tx_fill_seg_desc(volatile union gve_tx_desc *desc, uint64_t ol_flags,
+ union gve_tx_offload tx_offload,
+ uint16_t len, uint64_t addr)
+{
+ desc->seg.type_flags = GVE_TXD_SEG;
+ if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+ if (ol_flags & RTE_MBUF_F_TX_IPV6)
+ desc->seg.type_flags |= GVE_TXSF_IPV6;
+ desc->seg.l3_offset = tx_offload.l2_len >> 1;
+ desc->seg.mss = rte_cpu_to_be_16(tx_offload.tso_segsz);
+ }
+ desc->seg.seg_len = rte_cpu_to_be_16(len);
+ desc->seg.seg_addr = rte_cpu_to_be_64(addr);
+}
+
+static inline bool
+is_fifo_avail(struct gve_tx_queue *txq, uint16_t len)
+{
+ if (txq->fifo_avail < len)
+ return false;
+ /* Don't split segment. */
+ if (txq->fifo_head + len > txq->fifo_size &&
+ txq->fifo_size - txq->fifo_head + len > txq->fifo_avail)
+ return false;
+ return true;
+}
+static inline uint64_t
+gve_tx_alloc_from_fifo(struct gve_tx_queue *txq, uint16_t tx_id, uint16_t len)
+{
+ uint32_t head = txq->fifo_head;
+ uint32_t size = txq->fifo_size;
+ struct gve_tx_iovec *iov;
+ uint32_t aligned_head;
+ uint32_t iov_len = 0;
+ uint64_t fifo_addr;
+
+ iov = &txq->iov_ring[tx_id];
+
+ /* Don't split segment */
+ if (head + len > size) {
+ iov_len += (size - head);
+ head = 0;
+ }
+
+ fifo_addr = head;
+ iov_len += len;
+ iov->iov_base = head;
+
+ /* Re-align to a cacheline for next head */
+ head += len;
+ aligned_head = RTE_ALIGN(head, RTE_CACHE_LINE_SIZE);
+ iov_len += (aligned_head - head);
+ iov->iov_len = iov_len;
+
+ if (aligned_head == txq->fifo_size)
+ aligned_head = 0;
+ txq->fifo_head = aligned_head;
+ txq->fifo_avail -= iov_len;
+
+ return fifo_addr;
+}
+
+static inline uint16_t
+gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ union gve_tx_offload tx_offload = {0};
+ volatile union gve_tx_desc *txr, *txd;
+ struct gve_tx_queue *txq = tx_queue;
+ struct rte_mbuf **sw_ring = txq->sw_ring;
+ uint16_t mask = txq->nb_tx_desc - 1;
+ uint16_t tx_id = txq->tx_tail & mask;
+ uint64_t ol_flags, addr, fifo_addr;
+ uint32_t tx_tail = txq->tx_tail;
+ struct rte_mbuf *tx_pkt, *first;
+ uint16_t sw_id = txq->sw_tail;
+ uint16_t nb_used, i;
+ uint16_t nb_tx = 0;
+ uint32_t hlen;
+
+ txr = txq->tx_desc_ring;
+
+ if (txq->nb_free < txq->free_thresh || txq->fifo_avail == 0)
+ gve_tx_clean(txq);
+
+ if (txq->sw_nb_free < txq->free_thresh)
+ gve_tx_clean_swr_qpl(txq);
+
+ for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+ tx_pkt = *tx_pkts++;
+ ol_flags = tx_pkt->ol_flags;
+
+ if (txq->sw_nb_free < tx_pkt->nb_segs) {
+ gve_tx_clean_swr_qpl(txq);
+ if (txq->sw_nb_free < tx_pkt->nb_segs)
+ goto end_of_tx;
+ }
+
+ /* Even for multi-segs, use 1 qpl buf for data */
+ nb_used = 1;
+ if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
+ nb_used++;
+
+ if (txq->nb_free < nb_used)
+ goto end_of_tx;
+
+ tx_offload.l2_len = tx_pkt->l2_len;
+ tx_offload.l3_len = tx_pkt->l3_len;
+ tx_offload.l4_len = tx_pkt->l4_len;
+ tx_offload.tso_segsz = tx_pkt->tso_segsz;
+
+ first = tx_pkt;
+ txd = &txr[tx_id];
+ hlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?
+ (uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :
+ tx_pkt->pkt_len;
+
+ sw_ring[sw_id] = tx_pkt;
+ if (!is_fifo_avail(txq, hlen)) {
+ gve_tx_clean(txq);
+ if (!is_fifo_avail(txq, hlen))
+ goto end_of_tx;
+ }
+ addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off;
+ fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);
+
+ /* For TSO, check if there's enough fifo space for data first */
+ if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+ if (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen)) {
+ gve_tx_clean(txq);
+ if (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen))
+ goto end_of_tx;
+ }
+ }
+ if (tx_pkt->nb_segs == 1 || ol_flags & RTE_MBUF_F_TX_TCP_SEG)
+ rte_memcpy((void *)(fifo_addr + txq->fifo_base), (void *)addr, hlen);
+ else
+ rte_pktmbuf_read(tx_pkt, 0, hlen, (void *)(fifo_addr + txq->fifo_base));
+ gve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, fifo_addr);
+
+ if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+ tx_id = (tx_id + 1) & mask;
+ txd = &txr[tx_id];
+ addr = (uint64_t)(tx_pkt->buf_addr) + tx_pkt->data_off + hlen;
+ fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, tx_pkt->pkt_len - hlen);
+ if (tx_pkt->nb_segs == 1)
+ rte_memcpy((void *)(fifo_addr + txq->fifo_base), (void *)addr,
+ tx_pkt->pkt_len - hlen);
+ else
+ rte_pktmbuf_read(tx_pkt, hlen, tx_pkt->pkt_len - hlen,
+ (void *)(fifo_addr + txq->fifo_base));
+
+ gve_tx_fill_seg_desc(txd, ol_flags, tx_offload,
+ tx_pkt->pkt_len - hlen, fifo_addr);
+ }
+
+ /* record mbuf in sw_ring for free */
+ for (i = 1; i < first->nb_segs; i++) {
+ sw_id = (sw_id + 1) & mask;
+ tx_pkt = tx_pkt->next;
+ sw_ring[sw_id] = tx_pkt;
+ }
+
+ sw_id = (sw_id + 1) & mask;
+ tx_id = (tx_id + 1) & mask;
+
+ txq->nb_free -= nb_used;
+ txq->sw_nb_free -= first->nb_segs;
+ tx_tail += nb_used;
+ }
+
+end_of_tx:
+ if (nb_tx) {
+ rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);
+ txq->tx_tail = tx_tail;
+ txq->sw_tail = sw_id;
+ }
+
+ return nb_tx;
+}
+
+static inline uint16_t
+gve_tx_burst_ra(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ union gve_tx_offload tx_offload = {0};
+ volatile union gve_tx_desc *txr, *txd;
+ struct gve_tx_queue *txq = tx_queue;
+ struct rte_mbuf **sw_ring = txq->sw_ring;
+ uint16_t mask = txq->nb_tx_desc - 1;
+ uint16_t tx_id = txq->tx_tail & mask;
+ uint32_t tx_tail = txq->tx_tail;
+ struct rte_mbuf *tx_pkt, *first;
+ uint16_t nb_used, hlen, i;
+ uint64_t ol_flags, addr;
+ uint16_t nb_tx = 0;
+
+ txr = txq->tx_desc_ring;
+
+ if (txq->nb_free < txq->free_thresh)
+ gve_tx_clean(txq);
+
+ for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+ tx_pkt = *tx_pkts++;
+ ol_flags = tx_pkt->ol_flags;
+
+ nb_used = tx_pkt->nb_segs;
+ if (ol_flags & RTE_MBUF_F_TX_TCP_SEG)
+ nb_used++;
+
+ if (txq->nb_free < nb_used)
+ goto end_of_tx;
+
+ tx_offload.l2_len = tx_pkt->l2_len;
+ tx_offload.l3_len = tx_pkt->l3_len;
+ tx_offload.l4_len = tx_pkt->l4_len;
+ tx_offload.tso_segsz = tx_pkt->tso_segsz;
+
+ first = tx_pkt;
+ txd = &txr[tx_id];
+
+ hlen = ol_flags & RTE_MBUF_F_TX_TCP_SEG ?
+ (uint32_t)(tx_offload.l2_len + tx_offload.l3_len + tx_offload.l4_len) :
+ tx_pkt->pkt_len;
+ /*
+ * if tso, the driver needs to fill 2 descs for 1 mbuf
+ * so only put this mbuf into the 1st tx entry in sw ring
+ */
+ sw_ring[tx_id] = tx_pkt;
+ addr = rte_mbuf_data_iova(tx_pkt);
+ gve_tx_fill_pkt_desc(txd, tx_pkt, nb_used, hlen, addr);
+
+ if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
+ tx_id = (tx_id + 1) & mask;
+ txd = &txr[tx_id];
+ addr = rte_mbuf_data_iova(tx_pkt) + hlen;
+ gve_tx_fill_seg_desc(txd, ol_flags, tx_offload,
+ tx_pkt->data_len - hlen, addr);
+ }
+
+ for (i = 1; i < first->nb_segs; i++) {
+ tx_id = (tx_id + 1) & mask;
+ txd = &txr[tx_id];
+ tx_pkt = tx_pkt->next;
+ sw_ring[tx_id] = tx_pkt;
+ addr = rte_mbuf_data_iova(tx_pkt);
+ gve_tx_fill_seg_desc(txd, ol_flags, tx_offload,
+ tx_pkt->data_len, addr);
+ }
+ tx_id = (tx_id + 1) & mask;
+
+ txq->nb_free -= nb_used;
+ tx_tail += nb_used;
+ }
+
+end_of_tx:
+ if (nb_tx) {
+ rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);
+ txq->tx_tail = tx_tail;
+ }
+
+ return nb_tx;
+}
+
+uint16_t
+gve_tx_burst(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ struct gve_tx_queue *txq = tx_queue;
+
+ if (txq->is_gqi_qpl)
+ return gve_tx_burst_qpl(tx_queue, tx_pkts, nb_pkts);
+
+ return gve_tx_burst_ra(tx_queue, tx_pkts, nb_pkts);
+}
+
static inline void
gve_reset_txq(struct gve_tx_queue *txq)
{