@@ -298,6 +298,7 @@ gve_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
dev_info->default_txconf = (struct rte_eth_txconf) {
.tx_free_thresh = GVE_DEFAULT_TX_FREE_THRESH,
+ .tx_rs_thresh = GVE_DEFAULT_TX_RS_THRESH,
.offloads = 0,
};
@@ -528,6 +529,21 @@ static const struct eth_dev_ops gve_eth_dev_ops = {
.xstats_get_names = gve_xstats_get_names,
};
+static const struct eth_dev_ops gve_eth_dev_ops_dqo = {
+ .dev_configure = gve_dev_configure,
+ .dev_start = gve_dev_start,
+ .dev_stop = gve_dev_stop,
+ .dev_close = gve_dev_close,
+ .dev_infos_get = gve_dev_info_get,
+ .tx_queue_setup = gve_tx_queue_setup_dqo,
+ .link_update = gve_link_update,
+ .stats_get = gve_dev_stats_get,
+ .stats_reset = gve_dev_stats_reset,
+ .mtu_set = gve_dev_mtu_set,
+ .xstats_get = gve_xstats_get,
+ .xstats_get_names = gve_xstats_get_names,
+};
+
static void
gve_free_counter_array(struct gve_priv *priv)
{
@@ -770,8 +786,6 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
rte_be32_t *db_bar;
int err;
- eth_dev->dev_ops = &gve_eth_dev_ops;
-
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return 0;
@@ -807,10 +821,11 @@ gve_dev_init(struct rte_eth_dev *eth_dev)
return err;
if (gve_is_gqi(priv)) {
+ eth_dev->dev_ops = &gve_eth_dev_ops;
eth_dev->rx_pkt_burst = gve_rx_burst;
eth_dev->tx_pkt_burst = gve_tx_burst;
} else {
- PMD_DRV_LOG(ERR, "DQO_RDA is not implemented and will be added in the future");
+ eth_dev->dev_ops = &gve_eth_dev_ops_dqo;
}
eth_dev->data->mac_addrs = &priv->dev_addr;
@@ -28,7 +28,8 @@
#define PCI_MSIX_FLAGS_QSIZE 0x07FF /* Table size */
#define GVE_DEFAULT_RX_FREE_THRESH 512
-#define GVE_DEFAULT_TX_FREE_THRESH 256
+#define GVE_DEFAULT_TX_FREE_THRESH 32
+#define GVE_DEFAULT_TX_RS_THRESH 32
#define GVE_TX_MAX_FREE_SZ 512
#define GVE_MIN_BUF_SIZE 1024
@@ -53,6 +54,13 @@ union gve_tx_desc {
struct gve_tx_seg_desc seg; /* subsequent descs for a packet */
};
+/* Tx desc for DQO format */
+union gve_tx_desc_dqo {
+ struct gve_tx_pkt_desc_dqo pkt;
+ struct gve_tx_tso_context_desc_dqo tso_ctx;
+ struct gve_tx_general_context_desc_dqo general_ctx;
+};
+
/* Offload features */
union gve_tx_offload {
uint64_t data;
@@ -100,8 +108,10 @@ struct gve_tx_queue {
uint32_t tx_tail;
uint16_t nb_tx_desc;
uint16_t nb_free;
+ uint16_t nb_used;
uint32_t next_to_clean;
uint16_t free_thresh;
+ uint16_t rs_thresh;
/* Only valid for DQO_QPL queue format */
uint16_t sw_tail;
@@ -128,7 +138,15 @@ struct gve_tx_queue {
struct gve_queue_resources *qres;
/* newly added for DQO */
+ volatile union gve_tx_desc_dqo *tx_ring;
+ struct gve_tx_compl_desc *compl_ring;
+ const struct rte_memzone *compl_ring_mz;
uint64_t compl_ring_phys_addr;
+ uint32_t complq_tail;
+ uint16_t sw_size;
+ uint8_t cur_gen_bit;
+ uint32_t last_desc_cleaned;
+ void **txqs;
/* Only valid for DQO_RDA queue format */
struct gve_tx_queue *complq;
@@ -342,4 +360,11 @@ gve_rx_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
uint16_t
gve_tx_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
+/* Below functions are used for DQO */
+
+int
+gve_tx_queue_setup_dqo(struct rte_eth_dev *dev, uint16_t queue_id,
+ uint16_t nb_desc, unsigned int socket_id,
+ const struct rte_eth_txconf *conf);
+
#endif /* _GVE_ETHDEV_H_ */
new file mode 100644
@@ -0,0 +1,185 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022-2023 Google LLC
+ * Copyright (c) 2022-2023 Intel Corporation
+ */
+
+#include "gve_ethdev.h"
+#include "base/gve_adminq.h"
+
+static int
+check_tx_thresh_dqo(uint16_t nb_desc, uint16_t tx_rs_thresh,
+ uint16_t tx_free_thresh)
+{
+ if (tx_rs_thresh >= (nb_desc - 2)) {
+ PMD_DRV_LOG(ERR, "tx_rs_thresh (%u) must be less than the "
+ "number of TX descriptors (%u) minus 2",
+ tx_rs_thresh, nb_desc);
+ return -EINVAL;
+ }
+ if (tx_free_thresh >= (nb_desc - 3)) {
+ PMD_DRV_LOG(ERR, "tx_free_thresh (%u) must be less than the "
+ "number of TX descriptors (%u) minus 3.",
+ tx_free_thresh, nb_desc);
+ return -EINVAL;
+ }
+ if (tx_rs_thresh > tx_free_thresh) {
+ PMD_DRV_LOG(ERR, "tx_rs_thresh (%u) must be less than or "
+ "equal to tx_free_thresh (%u).",
+ tx_rs_thresh, tx_free_thresh);
+ return -EINVAL;
+ }
+ if ((nb_desc % tx_rs_thresh) != 0) {
+ PMD_DRV_LOG(ERR, "tx_rs_thresh (%u) must be a divisor of the "
+ "number of TX descriptors (%u).",
+ tx_rs_thresh, nb_desc);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void
+gve_reset_txq_dqo(struct gve_tx_queue *txq)
+{
+ struct rte_mbuf **sw_ring;
+ uint32_t size, i;
+
+ if (txq == NULL) {
+ PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
+ return;
+ }
+
+ size = txq->nb_tx_desc * sizeof(union gve_tx_desc_dqo);
+ for (i = 0; i < size; i++)
+ ((volatile char *)txq->tx_ring)[i] = 0;
+
+ size = txq->sw_size * sizeof(struct gve_tx_compl_desc);
+ for (i = 0; i < size; i++)
+ ((volatile char *)txq->compl_ring)[i] = 0;
+
+ sw_ring = txq->sw_ring;
+ for (i = 0; i < txq->sw_size; i++)
+ sw_ring[i] = NULL;
+
+ txq->tx_tail = 0;
+ txq->nb_used = 0;
+
+ txq->last_desc_cleaned = 0;
+ txq->sw_tail = 0;
+ txq->nb_free = txq->nb_tx_desc - 1;
+
+ txq->complq_tail = 0;
+ txq->cur_gen_bit = 1;
+}
+
+int
+gve_tx_queue_setup_dqo(struct rte_eth_dev *dev, uint16_t queue_id,
+ uint16_t nb_desc, unsigned int socket_id,
+ const struct rte_eth_txconf *conf)
+{
+ struct gve_priv *hw = dev->data->dev_private;
+ const struct rte_memzone *mz;
+ struct gve_tx_queue *txq;
+ uint16_t free_thresh;
+ uint16_t rs_thresh;
+ uint16_t sw_size;
+ int err = 0;
+
+ if (nb_desc != hw->tx_desc_cnt) {
+ PMD_DRV_LOG(WARNING, "gve doesn't support nb_desc config, use hw nb_desc %u.",
+ hw->tx_desc_cnt);
+ }
+ nb_desc = hw->tx_desc_cnt;
+
+ /* Allocate the TX queue data structure. */
+ txq = rte_zmalloc_socket("gve txq",
+ sizeof(struct gve_tx_queue),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (txq == NULL) {
+ PMD_DRV_LOG(ERR, "Failed to allocate memory for tx queue structure");
+ return -ENOMEM;
+ }
+
+ /* need to check free_thresh here */
+ free_thresh = conf->tx_free_thresh ?
+ conf->tx_free_thresh : GVE_DEFAULT_TX_FREE_THRESH;
+ rs_thresh = conf->tx_rs_thresh ?
+ conf->tx_rs_thresh : GVE_DEFAULT_TX_RS_THRESH;
+ if (check_tx_thresh_dqo(nb_desc, rs_thresh, free_thresh))
+ return -EINVAL;
+
+ txq->nb_tx_desc = nb_desc;
+ txq->free_thresh = free_thresh;
+ txq->rs_thresh = rs_thresh;
+ txq->queue_id = queue_id;
+ txq->port_id = dev->data->port_id;
+ txq->ntfy_id = queue_id;
+ txq->hw = hw;
+ txq->ntfy_addr = &hw->db_bar2[rte_be_to_cpu_32(hw->irq_dbs[txq->ntfy_id].id)];
+
+ /* Allocate software ring */
+ sw_size = nb_desc * DQO_TX_MULTIPLIER;
+ txq->sw_ring = rte_zmalloc_socket("gve tx sw ring",
+ sw_size * sizeof(struct rte_mbuf *),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (txq->sw_ring == NULL) {
+ PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring");
+ err = -ENOMEM;
+ goto free_txq;
+ }
+ txq->sw_size = sw_size;
+
+ /* Allocate TX hardware ring descriptors. */
+ mz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_id,
+ nb_desc * sizeof(union gve_tx_desc_dqo),
+ PAGE_SIZE, socket_id);
+ if (mz == NULL) {
+ PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX");
+ err = -ENOMEM;
+ goto free_txq_sw_ring;
+ }
+ txq->tx_ring = (union gve_tx_desc_dqo *)mz->addr;
+ txq->tx_ring_phys_addr = mz->iova;
+ txq->mz = mz;
+
+ /* Allocate TX completion ring descriptors. */
+ mz = rte_eth_dma_zone_reserve(dev, "tx_compl_ring", queue_id,
+ sw_size * sizeof(struct gve_tx_compl_desc),
+ PAGE_SIZE, socket_id);
+ if (mz == NULL) {
+ PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX completion queue");
+ err = -ENOMEM;
+ goto free_txq_mz;
+ }
+ txq->compl_ring = (struct gve_tx_compl_desc *)mz->addr;
+ txq->compl_ring_phys_addr = mz->iova;
+ txq->compl_ring_mz = mz;
+ txq->txqs = dev->data->tx_queues;
+
+ mz = rte_eth_dma_zone_reserve(dev, "txq_res", queue_id,
+ sizeof(struct gve_queue_resources),
+ PAGE_SIZE, socket_id);
+ if (mz == NULL) {
+ PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX resource");
+ err = -ENOMEM;
+ goto free_txq_cq_mz;
+ }
+ txq->qres = (struct gve_queue_resources *)mz->addr;
+ txq->qres_mz = mz;
+
+ gve_reset_txq_dqo(txq);
+
+ dev->data->tx_queues[queue_id] = txq;
+
+ return 0;
+
+free_txq_cq_mz:
+ rte_memzone_free(txq->compl_ring_mz);
+free_txq_mz:
+ rte_memzone_free(txq->mz);
+free_txq_sw_ring:
+ rte_free(txq->sw_ring);
+free_txq:
+ rte_free(txq);
+ return err;
+}
@@ -11,6 +11,7 @@ sources = files(
'base/gve_adminq.c',
'gve_rx.c',
'gve_tx.c',
+ 'gve_tx_dqo.c',
'gve_ethdev.c',
)
includes += include_directories('base')