@@ -378,6 +378,29 @@ int spnic_set_port_enable(void *hwdev, bool enable)
return 0;
}
+int spnic_flush_qps_res(void *hwdev)
+{
+ struct spnic_cmd_clear_qp_resource sq_res;
+ u16 out_size = sizeof(sq_res);
+ int err;
+
+ if (!hwdev)
+ return -EINVAL;
+
+ memset(&sq_res, 0, sizeof(sq_res));
+ sq_res.func_id = spnic_global_func_id(hwdev);
+
+ err = spnic_l2nic_msg_to_mgmt_sync(hwdev, SPNIC_CMD_CLEAR_QP_RESOURCE, &sq_res,
+ sizeof(sq_res), &sq_res, &out_size);
+ if (err || !out_size || sq_res.msg_head.status) {
+ PMD_DRV_LOG(ERR, "Clear sq resources failed, err: %d, status: 0x%x, out size: 0x%x",
+ err, sq_res.msg_head.status, out_size);
+ return -EIO;
+ }
+
+ return 0;
+}
+
static int spnic_set_function_table(void *hwdev, u32 cfg_bitmap,
struct spnic_func_tbl_cfg *cfg)
{
@@ -255,6 +255,17 @@ struct spnic_cmd_register_vf {
u8 rsvd[39];
};
+
+struct spnic_cmd_set_rq_flush {
+ union {
+ struct {
+ u16 global_rq_id;
+ u16 local_rq_id;
+ };
+ u32 value;
+ };
+};
+
int spnic_l2nic_msg_to_mgmt_sync(void *hwdev, u16 cmd, void *buf_in, u16 in_size,
void *buf_out, u16 *out_size);
@@ -381,6 +392,18 @@ int spnic_set_port_enable(void *hwdev, bool enable);
*/
int spnic_get_link_state(void *hwdev, u8 *link_state);
+/**
+ * Flush queue pairs resource in hardware
+ *
+ * @param[in] hwdev
+ * Device pointer to hwdev
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+int spnic_flush_qps_res(void *hwdev);
+
+
/**
* Init nic hwdev
*
@@ -13,6 +13,8 @@ objs = [base_objs]
sources = files(
'spnic_ethdev.c',
'spnic_io.c',
+ 'spnic_rx.c',
+ 'spnic_tx.c'
)
includes += include_directories('base')
@@ -139,6 +139,468 @@ static int spnic_link_update(struct rte_eth_dev *dev, int wait_to_complete)
return rte_eth_linkstatus_set(dev, &link);
}
+static void spnic_reset_rx_queue(struct rte_eth_dev *dev)
+{
+ struct spnic_rxq *rxq = NULL;
+ struct spnic_nic_dev *nic_dev;
+ int q_id = 0;
+
+ nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+
+ for (q_id = 0; q_id < nic_dev->num_rqs; q_id++) {
+ rxq = nic_dev->rxqs[q_id];
+
+ rxq->cons_idx = 0;
+ rxq->prod_idx = 0;
+ rxq->delta = rxq->q_depth;
+ rxq->next_to_update = 0;
+ }
+}
+
+static void spnic_reset_tx_queue(struct rte_eth_dev *dev)
+{
+ struct spnic_nic_dev *nic_dev;
+ struct spnic_txq *txq = NULL;
+ int q_id = 0;
+
+ nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+
+ for (q_id = 0; q_id < nic_dev->num_sqs; q_id++) {
+ txq = nic_dev->txqs[q_id];
+
+ txq->cons_idx = 0;
+ txq->prod_idx = 0;
+ txq->owner = 1;
+
+ /* Clear hardware ci */
+ *(u16 *)txq->ci_vaddr_base = 0;
+ }
+}
+
+/**
+ * Create the receive queue.
+ *
+ * @param[in] dev
+ * Pointer to ethernet device structure.
+ * @param[in] qid
+ * Receive queue index.
+ * @param[in] nb_desc
+ * Number of descriptors for receive queue.
+ * @param[in] socket_id
+ * Socket index on which memory must be allocated.
+ * @param rx_conf
+ * Thresholds parameters (unused_).
+ * @param mp
+ * Memory pool for buffer allocations.
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+static int spnic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid,
+ uint16_t nb_desc, unsigned int socket_id,
+ __rte_unused const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mp)
+{
+ struct spnic_nic_dev *nic_dev;
+ struct spnic_rxq *rxq = NULL;
+ const struct rte_memzone *rq_mz = NULL;
+ const struct rte_memzone *cqe_mz = NULL;
+ const struct rte_memzone *pi_mz = NULL;
+ u16 rq_depth, rx_free_thresh;
+ u32 queue_buf_size, mb_buf_size;
+ void *db_addr = NULL;
+ int wqe_count;
+ u32 buf_size;
+ int err;
+
+ nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+
+ /* Queue depth must be power of 2, otherwise will be aligned up */
+ rq_depth = (nb_desc & (nb_desc - 1)) ?
+ ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
+
+ /*
+ * Validate number of receive descriptors.
+ * It must not exceed hardware maximum and minimum.
+ */
+ if (rq_depth > SPNIC_MAX_QUEUE_DEPTH ||
+ rq_depth < SPNIC_MIN_QUEUE_DEPTH) {
+ PMD_DRV_LOG(ERR, "RX queue depth is out of range from %d to %d,"
+ "(nb_desc: %d, q_depth: %d, port: %d queue: %d)",
+ SPNIC_MIN_QUEUE_DEPTH, SPNIC_MAX_QUEUE_DEPTH,
+ (int)nb_desc, (int)rq_depth,
+ (int)dev->data->port_id, (int)qid);
+ return -EINVAL;
+ }
+
+ /*
+ * The RX descriptor ring will be cleaned after rxq->rx_free_thresh
+ * descriptors are used or if the number of descriptors required
+ * to transmit a packet is greater than the number of free RX
+ * descriptors.
+ * The following constraints must be satisfied:
+ * -rx_free_thresh must be greater than 0.
+ * -rx_free_thresh must be less than the size of the ring minus 1.
+ * When set to zero use default values.
+ */
+ rx_free_thresh = (u16)((rx_conf->rx_free_thresh) ?
+ rx_conf->rx_free_thresh : SPNIC_DEFAULT_RX_FREE_THRESH);
+ if (rx_free_thresh >= (rq_depth - 1)) {
+ PMD_DRV_LOG(ERR, "rx_free_thresh must be less than the number "
+ "of RX descriptors minus 1, rx_free_thresh: %u port: %d queue: %d)",
+ (unsigned int)rx_free_thresh,
+ (int)dev->data->port_id, (int)qid);
+ return -EINVAL;
+ }
+
+ rxq = rte_zmalloc_socket("spnic_rq", sizeof(struct spnic_rxq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (!rxq) {
+ PMD_DRV_LOG(ERR, "Allocate rxq[%d] failed, dev_name: %s",
+ qid, dev->data->name);
+ return -ENOMEM;
+ }
+
+ /* Init rq parameters */
+ rxq->nic_dev = nic_dev;
+ nic_dev->rxqs[qid] = rxq;
+ rxq->mb_pool = mp;
+ rxq->q_id = qid;
+ rxq->next_to_update = 0;
+ rxq->q_depth = rq_depth;
+ rxq->q_mask = rq_depth - 1;
+ rxq->delta = rq_depth;
+ rxq->cons_idx = 0;
+ rxq->prod_idx = 0;
+ rxq->wqe_type = SPNIC_NORMAL_RQ_WQE;
+ rxq->wqebb_shift = SPNIC_RQ_WQEBB_SHIFT + rxq->wqe_type;
+ rxq->wqebb_size = (u16)BIT(rxq->wqebb_shift);
+ rxq->rx_free_thresh = rx_free_thresh;
+ rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh;
+ rxq->port_id = dev->data->port_id;
+
+ /* If buf_len used for function table, need to translated */
+ mb_buf_size = rte_pktmbuf_data_room_size(rxq->mb_pool) -
+ RTE_PKTMBUF_HEADROOM;
+ err = spnic_convert_rx_buf_size(mb_buf_size, &buf_size);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Adjust buf size failed, dev_name: %s",
+ dev->data->name);
+ goto adjust_bufsize_fail;
+ }
+
+ rxq->buf_len = buf_size;
+ rxq->rx_buff_shift = ilog2(rxq->buf_len);
+
+ pi_mz = rte_eth_dma_zone_reserve(dev, "spnic_rq_pi", qid,
+ RTE_PGSIZE_4K, RTE_CACHE_LINE_SIZE,
+ socket_id);
+ if (!pi_mz) {
+ PMD_DRV_LOG(ERR, "Allocate rxq[%d] pi_mz failed, dev_name: %s",
+ qid, dev->data->name);
+ err = -ENOMEM;
+ goto alloc_pi_mz_fail;
+ }
+ rxq->pi_mz = pi_mz;
+ rxq->pi_dma_addr = pi_mz->iova;
+ rxq->pi_virt_addr = pi_mz->addr;
+
+ /* Rxq doesn't use direct wqe */
+ err = spnic_alloc_db_addr(nic_dev->hwdev, &db_addr, NULL);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Alloc rq doorbell addr failed");
+ goto alloc_db_err_fail;
+ }
+ rxq->db_addr = db_addr;
+
+ queue_buf_size = BIT(rxq->wqebb_shift) * rq_depth;
+ rq_mz = rte_eth_dma_zone_reserve(dev, "spnic_rq_mz", qid,
+ queue_buf_size, RTE_PGSIZE_256K,
+ socket_id);
+ if (!rq_mz) {
+ PMD_DRV_LOG(ERR, "Allocate rxq[%d] rq_mz failed, dev_name: %s",
+ qid, dev->data->name);
+ err = -ENOMEM;
+ goto alloc_rq_mz_fail;
+ }
+
+ memset(rq_mz->addr, 0, queue_buf_size);
+ rxq->rq_mz = rq_mz;
+ rxq->queue_buf_paddr = rq_mz->iova;
+ rxq->queue_buf_vaddr = rq_mz->addr;
+
+ rxq->rx_info = rte_zmalloc_socket("rx_info",
+ rq_depth * sizeof(*rxq->rx_info),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (!rxq->rx_info) {
+ PMD_DRV_LOG(ERR, "Allocate rx_info failed, dev_name: %s",
+ dev->data->name);
+ err = -ENOMEM;
+ goto alloc_rx_info_fail;
+ }
+
+ cqe_mz = rte_eth_dma_zone_reserve(dev, "spnic_cqe_mz", qid,
+ rq_depth * sizeof(*rxq->rx_cqe),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (!cqe_mz) {
+ PMD_DRV_LOG(ERR, "Allocate cqe mem zone failed, dev_name: %s",
+ dev->data->name);
+ err = -ENOMEM;
+ goto alloc_cqe_mz_fail;
+ }
+ memset(cqe_mz->addr, 0, rq_depth * sizeof(*rxq->rx_cqe));
+ rxq->cqe_mz = cqe_mz;
+ rxq->cqe_start_paddr = cqe_mz->iova;
+ rxq->cqe_start_vaddr = cqe_mz->addr;
+ rxq->rx_cqe = (struct spnic_rq_cqe *)rxq->cqe_start_vaddr;
+
+ wqe_count = spnic_rx_fill_wqe(rxq);
+ if (wqe_count != rq_depth) {
+ PMD_DRV_LOG(ERR, "Fill rx wqe failed, wqe_count: %d, dev_name: %s",
+ wqe_count, dev->data->name);
+ err = -ENOMEM;
+ goto fill_rx_wqe_fail;
+ }
+
+ /* Record rxq pointer in rte_eth rx_queues */
+ dev->data->rx_queues[qid] = rxq;
+
+ return 0;
+
+fill_rx_wqe_fail:
+ rte_memzone_free(rxq->cqe_mz);
+alloc_cqe_mz_fail:
+ rte_free(rxq->rx_info);
+
+alloc_rx_info_fail:
+ rte_memzone_free(rxq->rq_mz);
+
+alloc_rq_mz_fail:
+ spnic_free_db_addr(nic_dev->hwdev, rxq->db_addr, NULL);
+
+alloc_db_err_fail:
+ rte_memzone_free(rxq->pi_mz);
+
+alloc_pi_mz_fail:
+adjust_bufsize_fail:
+
+ rte_free(rxq);
+ nic_dev->rxqs[qid] = NULL;
+
+ return err;
+}
+
+/**
+ * Create the transmit queue.
+ *
+ * @param[in] dev
+ * Pointer to ethernet device structure.
+ * @param[in] queue_idx
+ * Transmit queue index.
+ * @param[in] nb_desc
+ * Number of descriptors for transmit queue.
+ * @param[in] socket_id
+ * Socket index on which memory must be allocated.
+ * @param[in] tx_conf
+ * Tx queue configuration parameters (unused_).
+ *
+ * @retval zero : Success
+ * @retval non-zero : Failure
+ */
+static int spnic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qid,
+ uint16_t nb_desc, unsigned int socket_id,
+ __rte_unused const struct rte_eth_txconf *tx_conf)
+{
+ struct spnic_nic_dev *nic_dev;
+ struct spnic_hwdev *hwdev;
+ struct spnic_txq *txq = NULL;
+ const struct rte_memzone *sq_mz = NULL;
+ const struct rte_memzone *ci_mz = NULL;
+ void *db_addr = NULL;
+ u16 sq_depth, tx_free_thresh;
+ u32 queue_buf_size;
+ int err;
+
+ nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
+ hwdev = nic_dev->hwdev;
+
+ /* Queue depth must be power of 2, otherwise will be aligned up */
+ sq_depth = (nb_desc & (nb_desc - 1)) ?
+ ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
+
+ /*
+ * Validate number of transmit descriptors.
+ * It must not exceed hardware maximum and minimum.
+ */
+ if (sq_depth > SPNIC_MAX_QUEUE_DEPTH ||
+ sq_depth < SPNIC_MIN_QUEUE_DEPTH) {
+ PMD_DRV_LOG(ERR, "TX queue depth is out of range from %d to %d,"
+ "(nb_desc: %d, q_depth: %d, port: %d queue: %d)",
+ SPNIC_MIN_QUEUE_DEPTH, SPNIC_MAX_QUEUE_DEPTH,
+ (int)nb_desc, (int)sq_depth,
+ (int)dev->data->port_id, (int)qid);
+ return -EINVAL;
+ }
+
+ /*
+ * The TX descriptor ring will be cleaned after txq->tx_free_thresh
+ * descriptors are used or if the number of descriptors required
+ * to transmit a packet is greater than the number of free TX
+ * descriptors.
+ * The following constraints must be satisfied:
+ * -tx_free_thresh must be greater than 0.
+ * -tx_free_thresh must be less than the size of the ring minus 1.
+ * When set to zero use default values.
+ */
+ tx_free_thresh = (u16)((tx_conf->tx_free_thresh) ?
+ tx_conf->tx_free_thresh : SPNIC_DEFAULT_TX_FREE_THRESH);
+ if (tx_free_thresh >= (sq_depth - 1)) {
+ PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the number of tx "
+ "descriptors minus 1, tx_free_thresh: %u port: %d queue: %d",
+ (unsigned int)tx_free_thresh,
+ (int)dev->data->port_id, (int)qid);
+ return -EINVAL;
+ }
+
+ txq = rte_zmalloc_socket("spnic_tx_queue", sizeof(struct spnic_txq),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (!txq) {
+ PMD_DRV_LOG(ERR, "Allocate txq[%d] failed, dev_name: %s",
+ qid, dev->data->name);
+ return -ENOMEM;
+ }
+ nic_dev->txqs[qid] = txq;
+ txq->nic_dev = nic_dev;
+ txq->q_id = qid;
+ txq->q_depth = sq_depth;
+ txq->q_mask = sq_depth - 1;
+ txq->cons_idx = 0;
+ txq->prod_idx = 0;
+ txq->wqebb_shift = SPNIC_SQ_WQEBB_SHIFT;
+ txq->wqebb_size = (u16)BIT(txq->wqebb_shift);
+ txq->tx_free_thresh = tx_free_thresh;
+ txq->owner = 1;
+ txq->cos = nic_dev->default_cos;
+
+ ci_mz = rte_eth_dma_zone_reserve(dev, "spnic_sq_ci", qid,
+ SPNIC_CI_Q_ADDR_SIZE,
+ SPNIC_CI_Q_ADDR_SIZE, socket_id);
+ if (!ci_mz) {
+ PMD_DRV_LOG(ERR, "Allocate txq[%d] ci_mz failed, dev_name: %s",
+ qid, dev->data->name);
+ err = -ENOMEM;
+ goto alloc_ci_mz_fail;
+ }
+ txq->ci_mz = ci_mz;
+ txq->ci_dma_base = ci_mz->iova;
+ txq->ci_vaddr_base = ci_mz->addr;
+
+ queue_buf_size = BIT(txq->wqebb_shift) * sq_depth;
+ sq_mz = rte_eth_dma_zone_reserve(dev, "spnic_sq_mz", qid,
+ queue_buf_size, RTE_PGSIZE_256K,
+ socket_id);
+ if (!sq_mz) {
+ PMD_DRV_LOG(ERR, "Allocate txq[%d] sq_mz failed, dev_name: %s",
+ qid, dev->data->name);
+ err = -ENOMEM;
+ goto alloc_sq_mz_fail;
+ }
+ memset(sq_mz->addr, 0, queue_buf_size);
+ txq->sq_mz = sq_mz;
+ txq->queue_buf_paddr = sq_mz->iova;
+ txq->queue_buf_vaddr = sq_mz->addr;
+ txq->sq_head_addr = (u64)txq->queue_buf_vaddr;
+ txq->sq_bot_sge_addr = txq->sq_head_addr + queue_buf_size;
+
+ /* Sq doesn't use direct wqe */
+ err = spnic_alloc_db_addr(hwdev, &db_addr, NULL);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Alloc sq doorbell addr failed");
+ goto alloc_db_err_fail;
+ }
+ txq->db_addr = db_addr;
+
+ txq->tx_info = rte_zmalloc_socket("tx_info",
+ sq_depth * sizeof(*txq->tx_info),
+ RTE_CACHE_LINE_SIZE, socket_id);
+ if (!txq->tx_info) {
+ PMD_DRV_LOG(ERR, "Allocate tx_info failed, dev_name: %s",
+ dev->data->name);
+ err = -ENOMEM;
+ goto alloc_tx_info_fail;
+ }
+
+ /* Record txq pointer in rte_eth tx_queues */
+ dev->data->tx_queues[qid] = txq;
+
+ return 0;
+
+alloc_tx_info_fail:
+ spnic_free_db_addr(hwdev, txq->db_addr, NULL);
+
+alloc_db_err_fail:
+ rte_memzone_free(txq->sq_mz);
+
+alloc_sq_mz_fail:
+ rte_memzone_free(txq->ci_mz);
+
+alloc_ci_mz_fail:
+ rte_free(txq);
+
+ return err;
+}
+
+static void spnic_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct spnic_rxq *rxq = dev->data->rx_queues[qid];
+ struct spnic_nic_dev *nic_dev;
+
+ if (!rxq) {
+ PMD_DRV_LOG(WARNING, "Rxq is null when release");
+ return;
+ }
+ nic_dev = rxq->nic_dev;
+
+ spnic_free_rxq_mbufs(rxq);
+
+ rte_memzone_free(rxq->cqe_mz);
+
+ rte_free(rxq->rx_info);
+
+ rte_memzone_free(rxq->rq_mz);
+
+ rte_memzone_free(rxq->pi_mz);
+
+ nic_dev->rxqs[rxq->q_id] = NULL;
+ rte_free(rxq);
+}
+
+static void spnic_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
+{
+ struct spnic_txq *txq = dev->data->tx_queues[qid];
+ struct spnic_nic_dev *nic_dev;
+
+ if (!txq) {
+ PMD_DRV_LOG(WARNING, "Txq is null when release");
+ return;
+ }
+ nic_dev = txq->nic_dev;
+
+ spnic_free_txq_mbufs(txq);
+
+ rte_free(txq->tx_info);
+ txq->tx_info = NULL;
+
+ spnic_free_db_addr(nic_dev->hwdev, txq->db_addr, NULL);
+
+ rte_memzone_free(txq->sq_mz);
+
+ rte_memzone_free(txq->ci_mz);
+
+ nic_dev->txqs[txq->q_id] = NULL;
+ rte_free(txq);
+}
+
static void spnic_delete_mc_addr_list(struct spnic_nic_dev *nic_dev);
/**
@@ -235,6 +697,7 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev)
nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+ spnic_get_func_rx_buf_size(nic_dev);
err = spnic_init_function_table(nic_dev->hwdev, nic_dev->rx_buff_len);
if (err) {
PMD_DRV_LOG(ERR, "Init function table failed, dev_name: %s",
@@ -253,6 +716,9 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev)
goto get_feature_err;
}
+ /* reset rx and tx queue */
+ spnic_reset_rx_queue(eth_dev);
+ spnic_reset_tx_queue(eth_dev);
/* Init txq and rxq context */
err = spnic_init_qp_ctxts(nic_dev);
@@ -270,6 +736,15 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev)
goto set_mtu_fail;
}
+ err = spnic_start_all_rqs(eth_dev);
+ if (err) {
+ PMD_DRV_LOG(ERR, "Set rx config failed, dev_name: %s",
+ eth_dev->data->name);
+ goto start_rqs_fail;
+ }
+
+ spnic_start_all_sqs(eth_dev);
+
/* Update eth_dev link status */
if (eth_dev->data->dev_conf.intr_conf.lsc != 0)
(void)spnic_link_update(eth_dev, 0);
@@ -278,6 +753,7 @@ static int spnic_dev_start(struct rte_eth_dev *eth_dev)
return 0;
+start_rqs_fail:
set_mtu_fail:
spnic_free_qp_ctxts(nic_dev->hwdev);
@@ -313,9 +789,17 @@ static int spnic_dev_stop(struct rte_eth_dev *dev)
memset(&link, 0, sizeof(link));
(void)rte_eth_linkstatus_set(dev, &link);
+ /* Flush pending io request */
+ spnic_flush_txqs(nic_dev);
+
+ spnic_flush_qps_res(nic_dev->hwdev);
/* Clean root context */
spnic_free_qp_ctxts(nic_dev->hwdev);
+ /* Free all tx and rx mbufs */
+ spnic_free_all_txq_mbufs(nic_dev);
+ spnic_free_all_rxq_mbufs(nic_dev);
+
return 0;
}
@@ -329,6 +813,7 @@ static int spnic_dev_close(struct rte_eth_dev *eth_dev)
{
struct spnic_nic_dev *nic_dev =
SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+ int qid;
if (rte_bit_relaxed_test_and_set32(SPNIC_DEV_CLOSE, &nic_dev->dev_status)) {
PMD_DRV_LOG(WARNING, "Device %s already closed",
@@ -338,6 +823,13 @@ static int spnic_dev_close(struct rte_eth_dev *eth_dev)
spnic_dev_stop(eth_dev);
+ /* Release io resource */
+ for (qid = 0; qid < nic_dev->num_sqs; qid++)
+ spnic_tx_queue_release(eth_dev, qid);
+
+ for (qid = 0; qid < nic_dev->num_rqs; qid++)
+ spnic_rx_queue_release(eth_dev, qid);
+
spnic_deinit_sw_rxtxqs(nic_dev);
spnic_deinit_mac_addr(eth_dev);
rte_free(nic_dev->mc_list);
@@ -581,6 +1073,10 @@ static const struct eth_dev_ops spnic_pmd_ops = {
.dev_set_link_up = spnic_dev_set_link_up,
.dev_set_link_down = spnic_dev_set_link_down,
.link_update = spnic_link_update,
+ .rx_queue_setup = spnic_rx_queue_setup,
+ .tx_queue_setup = spnic_tx_queue_setup,
+ .rx_queue_release = spnic_rx_queue_release,
+ .tx_queue_release = spnic_tx_queue_release,
.dev_start = spnic_dev_start,
.dev_stop = spnic_dev_stop,
.dev_close = spnic_dev_close,
@@ -592,8 +1088,12 @@ static const struct eth_dev_ops spnic_pmd_ops = {
};
static const struct eth_dev_ops spnic_pmd_vf_ops = {
- .link_update = spnic_link_update,
+ .rx_queue_setup = spnic_rx_queue_setup,
+ .tx_queue_setup = spnic_tx_queue_setup,
.dev_start = spnic_dev_start,
+ .link_update = spnic_link_update,
+ .rx_queue_release = spnic_rx_queue_release,
+ .tx_queue_release = spnic_tx_queue_release,
.dev_stop = spnic_dev_stop,
.dev_close = spnic_dev_close,
.mtu_set = spnic_dev_set_mtu,
new file mode 100644
@@ -0,0 +1,283 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Ramaxel Memory Technology, Ltd
+ */
+
+#include <rte_ether.h>
+#include <rte_mbuf.h>
+#include <ethdev_driver.h>
+
+#include "base/spnic_compat.h"
+#include "base/spnic_cmd.h"
+#include "base/spnic_hwif.h"
+#include "base/spnic_hwdev.h"
+#include "base/spnic_wq.h"
+#include "base/spnic_mgmt.h"
+#include "base/spnic_nic_cfg.h"
+#include "spnic_io.h"
+#include "spnic_rx.h"
+#include "spnic_ethdev.h"
+
+/**
+ * Get receive queue wqe
+ *
+ * @param[in] rxq
+ * Receive queue
+ * @param[out] pi
+ * Return current pi
+ * @return
+ * RX wqe base address
+ */
+static inline void *spnic_get_rq_wqe(struct spnic_rxq *rxq, u16 *pi)
+{
+ *pi = MASKED_QUEUE_IDX(rxq, rxq->prod_idx);
+
+ /* Get only one rq wqe for once */
+ rxq->prod_idx++;
+ rxq->delta--;
+
+ return NIC_WQE_ADDR(rxq, *pi);
+}
+
+/**
+ * Put receive queue wqe
+ *
+ * @param[in] rxq
+ * Receive queue
+ * @param[in] wqe_cnt
+ * Wqebb counters
+ */
+static inline void spnic_put_rq_wqe(struct spnic_rxq *rxq, u16 wqe_cnt)
+{
+ rxq->delta += wqe_cnt;
+ rxq->prod_idx -= wqe_cnt;
+}
+
+/**
+ * Get receive queue local pi
+ *
+ * @param[in] rxq
+ * Receive queue
+ * @return
+ * Receive queue local pi
+ */
+static inline u16 spnic_get_rq_local_pi(struct spnic_rxq *rxq)
+{
+ return MASKED_QUEUE_IDX(rxq, rxq->prod_idx);
+}
+
+int spnic_rx_fill_wqe(struct spnic_rxq *rxq)
+{
+ struct spnic_rq_wqe *rq_wqe = NULL;
+ struct spnic_nic_dev *nic_dev = rxq->nic_dev;
+ rte_iova_t cqe_dma;
+ u16 pi = 0;
+ int i;
+
+ cqe_dma = rxq->cqe_start_paddr;
+ for (i = 0; i < rxq->q_depth; i++) {
+ rq_wqe = spnic_get_rq_wqe(rxq, &pi);
+ if (!rq_wqe) {
+ PMD_DRV_LOG(ERR, "Get rq wqe failed, rxq id: %d, wqe id: %d",
+ rxq->q_id, i);
+ break;
+ }
+
+ if (rxq->wqe_type == SPNIC_EXTEND_RQ_WQE) {
+ /* Unit of cqe length is 16B */
+ spnic_set_sge(&rq_wqe->extend_wqe.cqe_sect.sge,
+ cqe_dma,
+ sizeof(struct spnic_rq_cqe) >>
+ SPNIC_CQE_SIZE_SHIFT);
+ /* Use fixed len */
+ rq_wqe->extend_wqe.buf_desc.sge.len =
+ nic_dev->rx_buff_len;
+ } else {
+ rq_wqe->normal_wqe.cqe_hi_addr = upper_32_bits(cqe_dma);
+ rq_wqe->normal_wqe.cqe_lo_addr = lower_32_bits(cqe_dma);
+ }
+
+ cqe_dma += sizeof(struct spnic_rq_cqe);
+ }
+
+ spnic_put_rq_wqe(rxq, (u16)i);
+
+ return i;
+}
+
+static struct rte_mbuf *spnic_rx_alloc_mbuf(struct spnic_rxq *rxq,
+ rte_iova_t *dma_addr)
+{
+ struct rte_mbuf *mbuf = NULL;
+
+ if (unlikely(rte_pktmbuf_alloc_bulk(rxq->mb_pool, &mbuf, 1) != 0))
+ return NULL;
+
+ *dma_addr = rte_mbuf_data_iova_default(mbuf);
+
+ return mbuf;
+}
+
+u32 spnic_rx_fill_buffers(struct spnic_rxq *rxq)
+{
+ struct spnic_rq_wqe *rq_wqe = NULL;
+ struct spnic_rx_info *rx_info = NULL;
+ struct rte_mbuf *mb = NULL;
+ rte_iova_t dma_addr;
+ int i, free_wqebbs;
+
+ free_wqebbs = rxq->delta - 1;
+ for (i = 0; i < free_wqebbs; i++) {
+ rx_info = &rxq->rx_info[rxq->next_to_update];
+
+ mb = spnic_rx_alloc_mbuf(rxq, &dma_addr);
+ if (!mb) {
+ PMD_DRV_LOG(ERR, "Alloc mbuf failed");
+ break;
+ }
+
+ rx_info->mbuf = mb;
+
+ rq_wqe = NIC_WQE_ADDR(rxq, rxq->next_to_update);
+
+ /* Fill buffer address only */
+ if (rxq->wqe_type == SPNIC_EXTEND_RQ_WQE) {
+ rq_wqe->extend_wqe.buf_desc.sge.hi_addr = upper_32_bits(dma_addr);
+ rq_wqe->extend_wqe.buf_desc.sge.lo_addr = lower_32_bits(dma_addr);
+ } else {
+ rq_wqe->normal_wqe.buf_hi_addr = upper_32_bits(dma_addr);
+ rq_wqe->normal_wqe.buf_lo_addr = lower_32_bits(dma_addr);
+ }
+
+ rxq->next_to_update = (rxq->next_to_update + 1) & rxq->q_mask;
+ }
+
+ if (likely(i > 0)) {
+ spnic_write_db(rxq->db_addr, rxq->q_id, 0, RQ_CFLAG_DP,
+ rxq->next_to_update << rxq->wqe_type);
+ /* Init rq contxet used, need to optimization */
+ rxq->prod_idx = rxq->next_to_update;
+ rxq->delta -= i;
+ } else {
+ PMD_DRV_LOG(ERR, "Alloc rx buffers failed, rxq_id: %d",
+ rxq->q_id);
+ }
+
+ return i;
+}
+
+void spnic_free_rxq_mbufs(struct spnic_rxq *rxq)
+{
+ struct spnic_rx_info *rx_info = NULL;
+ int free_wqebbs = spnic_get_rq_free_wqebb(rxq) + 1;
+ volatile struct spnic_rq_cqe *rx_cqe = NULL;
+ u16 ci;
+
+ while (free_wqebbs++ < rxq->q_depth) {
+ ci = spnic_get_rq_local_ci(rxq);
+
+ rx_cqe = &rxq->rx_cqe[ci];
+
+ /* Clear done bit */
+ rx_cqe->status = 0;
+
+ rx_info = &rxq->rx_info[ci];
+ rte_pktmbuf_free(rx_info->mbuf);
+ rx_info->mbuf = NULL;
+
+ spnic_update_rq_local_ci(rxq, 1);
+ }
+}
+
+void spnic_free_all_rxq_mbufs(struct spnic_nic_dev *nic_dev)
+{
+ u16 qid;
+
+ for (qid = 0; qid < nic_dev->num_rqs; qid++)
+ spnic_free_rxq_mbufs(nic_dev->rxqs[qid]);
+}
+
+static inline u32 spnic_rx_alloc_mbuf_bulk(struct spnic_rxq *rxq,
+ struct rte_mbuf **mbufs,
+ u32 exp_mbuf_cnt)
+{
+ u32 avail_cnt;
+ int err;
+
+ err = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, exp_mbuf_cnt);
+ if (likely(err == 0)) {
+ avail_cnt = exp_mbuf_cnt;
+ } else {
+ avail_cnt = 0;
+ rxq->rxq_stats.rx_nombuf += exp_mbuf_cnt;
+ }
+
+ return avail_cnt;
+}
+
+static inline void spnic_rearm_rxq_mbuf(struct spnic_rxq *rxq)
+{
+ struct spnic_rq_wqe *rq_wqe = NULL;
+ struct rte_mbuf **rearm_mbufs;
+ u32 i, free_wqebbs, rearm_wqebbs, exp_wqebbs;
+ rte_iova_t dma_addr;
+ u16 pi;
+
+ /* Check free wqebb cnt for rearm */
+ free_wqebbs = spnic_get_rq_free_wqebb(rxq);
+ if (unlikely(free_wqebbs < rxq->rx_free_thresh))
+ return;
+
+ /* Get rearm mbuf array */
+ pi = spnic_get_rq_local_pi(rxq);
+ rearm_mbufs = (struct rte_mbuf **)(&rxq->rx_info[pi]);
+
+ /* Check rxq free wqebbs turn around */
+ exp_wqebbs = rxq->q_depth - pi;
+ if (free_wqebbs < exp_wqebbs)
+ exp_wqebbs = free_wqebbs;
+
+ /* Alloc mbuf in bulk */
+ rearm_wqebbs = spnic_rx_alloc_mbuf_bulk(rxq, rearm_mbufs, exp_wqebbs);
+ if (unlikely(rearm_wqebbs == 0))
+ return;
+
+ /* Rearm rx mbuf */
+ rq_wqe = NIC_WQE_ADDR(rxq, pi);
+ for (i = 0; i < rearm_wqebbs; i++) {
+ dma_addr = rte_mbuf_data_iova_default(rearm_mbufs[i]);
+
+ /* Fill buffer address only */
+ if (rxq->wqe_type == SPNIC_EXTEND_RQ_WQE) {
+ rq_wqe->extend_wqe.buf_desc.sge.hi_addr = upper_32_bits(dma_addr);
+ rq_wqe->extend_wqe.buf_desc.sge.lo_addr = lower_32_bits(dma_addr);
+ } else {
+ rq_wqe->normal_wqe.buf_hi_addr = upper_32_bits(dma_addr);
+ rq_wqe->normal_wqe.buf_lo_addr = lower_32_bits(dma_addr);
+ }
+
+ rq_wqe = (struct spnic_rq_wqe *)((u64)rq_wqe +
+ rxq->wqebb_size);
+ }
+ rxq->prod_idx += rearm_wqebbs;
+ rxq->delta -= rearm_wqebbs;
+
+ spnic_write_db(rxq->db_addr, rxq->q_id, 0, RQ_CFLAG_DP,
+ ((pi + rearm_wqebbs) & rxq->q_mask) << rxq->wqe_type);
+}
+
+int spnic_start_all_rqs(struct rte_eth_dev *eth_dev)
+{
+ struct spnic_nic_dev *nic_dev = NULL;
+ struct spnic_rxq *rxq = NULL;
+ int i;
+
+ nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+
+ for (i = 0; i < nic_dev->num_rqs; i++) {
+ rxq = eth_dev->data->rx_queues[i];
+ spnic_rearm_rxq_mbuf(rxq);
+ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+ }
+
+ return 0;
+}
@@ -110,4 +110,45 @@ struct spnic_rxq {
struct spnic_rxq_stats rxq_stats;
} __rte_cache_aligned;
+int spnic_rx_fill_wqe(struct spnic_rxq *rxq);
+
+u32 spnic_rx_fill_buffers(struct spnic_rxq *rxq);
+
+void spnic_free_rxq_mbufs(struct spnic_rxq *rxq);
+
+void spnic_free_all_rxq_mbufs(struct spnic_nic_dev *nic_dev);
+
+int spnic_start_all_rqs(struct rte_eth_dev *eth_dev);
+/**
+ * Get receive queue local ci
+ *
+ * @param[in] rxq
+ * Receive queue
+ * @return
+ * Receive queue local ci
+ */
+static inline u16 spnic_get_rq_local_ci(struct spnic_rxq *rxq)
+{
+ return MASKED_QUEUE_IDX(rxq, rxq->cons_idx);
+}
+
+static inline u16 spnic_get_rq_free_wqebb(struct spnic_rxq *rxq)
+{
+ return rxq->delta - 1;
+}
+
+/**
+ * Update receive queue local ci
+ *
+ * @param[in] rxq
+ * Receive queue
+ * @param[in] wqe_cnt
+ * Wqebb counters
+ */
+static inline void spnic_update_rq_local_ci(struct spnic_rxq *rxq,
+ u16 wqe_cnt)
+{
+ rxq->cons_idx += wqe_cnt;
+ rxq->delta += wqe_cnt;
+}
#endif /* _SPNIC_RX_H_ */
new file mode 100644
@@ -0,0 +1,334 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Ramaxel Memory Technology, Ltd
+ */
+
+#include <rte_ether.h>
+#include <rte_mbuf.h>
+#include <rte_io.h>
+#include <ethdev_driver.h>
+
+#include "base/spnic_compat.h"
+#include "base/spnic_cmd.h"
+#include "base/spnic_wq.h"
+#include "base/spnic_mgmt.h"
+#include "base/spnic_hwdev.h"
+#include "base/spnic_nic_cfg.h"
+#include "spnic_io.h"
+#include "spnic_tx.h"
+#include "spnic_ethdev.h"
+
+#define SPNIC_TX_TASK_WRAPPED 1
+#define SPNIC_TX_BD_DESC_WRAPPED 2
+
+#define TX_MSS_DEFAULT 0x3E00
+#define TX_MSS_MIN 0x50
+
+#define SPNIC_MAX_TX_FREE_BULK 64
+
+#define MAX_PAYLOAD_OFFSET 221
+
+#define SPNIC_TX_OUTER_CHECKSUM_FLAG_SET 1
+#define SPNIC_TX_OUTER_CHECKSUM_FLAG_NO_SET 0
+
+/**
+ * Get send queue free wqebb cnt
+ *
+ * @param[in] sq
+ * Send queue
+ * @return
+ * Number of free wqebb
+ */
+static inline u16 spnic_get_sq_free_wqebbs(struct spnic_txq *sq)
+{
+ return (sq->q_depth -
+ ((sq->prod_idx - sq->cons_idx + sq->q_depth) & sq->q_mask) - 1);
+}
+
+/**
+ * Update send queue local ci
+ *
+ * @param[in] sq
+ * Send queue
+ * @param[in] wqe_cnt
+ * Number of wqebb
+ */
+static inline void spnic_update_sq_local_ci(struct spnic_txq *sq, u16 wqe_cnt)
+{
+ sq->cons_idx += wqe_cnt;
+}
+
+/**
+ * Get send queue local ci
+ *
+ * @param[in] sq
+ * Send queue
+ * @return
+ * Local ci
+ */
+static inline u16 spnic_get_sq_local_ci(struct spnic_txq *sq)
+{
+ return MASKED_QUEUE_IDX(sq, sq->cons_idx);
+}
+
+/**
+ * Get send queue hardware ci
+ *
+ * @param[in] sq
+ * Send queue
+ * @return
+ * Hardware ci
+ */
+static inline u16 spnic_get_sq_hw_ci(struct spnic_txq *sq)
+{
+ return MASKED_QUEUE_IDX(sq, *(u16 *)sq->ci_vaddr_base);
+}
+
+/**
+ * Get send queue wqe
+ *
+ * @param[in] sq
+ * Send queue
+ * @param[in] wqebb_cnt
+ * Num of wqebb counter
+ * @param[out] pi
+ * Return current pi
+ * @param[out] owner
+ * Owner bit for hardware
+ * @param[out] wrapped
+ * Indicate whether wqe is wrapped
+ * @return
+ * Send queue wqe base address
+ */
+static inline void *spnic_get_sq_wqe(struct spnic_txq *sq,
+ struct spnic_wqe_info *wqe_info)
+{
+ u16 cur_pi = MASKED_QUEUE_IDX(sq, sq->prod_idx);
+ u32 end_pi;
+
+ end_pi = cur_pi + wqe_info->wqebb_cnt;
+ sq->prod_idx += wqe_info->wqebb_cnt;
+
+ wqe_info->owner = sq->owner;
+ wqe_info->pi = cur_pi;
+ wqe_info->wrapped = 0;
+
+ if (unlikely(end_pi >= sq->q_depth)) {
+ sq->owner = !sq->owner;
+
+ if (likely(end_pi > sq->q_depth))
+ wqe_info->wrapped = sq->q_depth - cur_pi;
+ }
+
+ return NIC_WQE_ADDR(sq, cur_pi);
+}
+
+/**
+ * Put send queue wqe
+ *
+ * @param[in] sq
+ * Send queue
+ * @param[in] wqebb_cnt
+ * Num of wqebb counter
+ * @param[out] owner
+ * Owner bit for hardware
+ */
+static inline void spnic_put_sq_wqe(struct spnic_txq *sq,
+ struct spnic_wqe_info *wqe_info)
+{
+ if (wqe_info->owner != sq->owner)
+ sq->owner = wqe_info->owner;
+
+ sq->prod_idx -= wqe_info->wqebb_cnt;
+}
+
+static inline void spnic_set_wqe_combo(struct spnic_txq *txq,
+ struct spnic_sq_wqe_combo *wqe_combo,
+ struct spnic_sq_wqe *wqe,
+ struct spnic_wqe_info *wqe_info)
+{
+ wqe_combo->hdr = &wqe->compact_wqe.wqe_desc;
+
+ if (wqe_info->offload) {
+ if (wqe_info->wrapped == SPNIC_TX_TASK_WRAPPED) {
+ wqe_combo->task = (struct spnic_sq_task *)
+ (void *)txq->sq_head_addr;
+ wqe_combo->bds_head = (struct spnic_sq_bufdesc *)
+ (void *)(txq->sq_head_addr + txq->wqebb_size);
+ } else if (wqe_info->wrapped == SPNIC_TX_BD_DESC_WRAPPED) {
+ wqe_combo->task = &wqe->extend_wqe.task;
+ wqe_combo->bds_head = (struct spnic_sq_bufdesc *)
+ (void *)(txq->sq_head_addr);
+ } else {
+ wqe_combo->task = &wqe->extend_wqe.task;
+ wqe_combo->bds_head = wqe->extend_wqe.buf_desc;
+ }
+
+ wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE;
+ wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES;
+ return;
+ }
+
+ if (wqe_info->wrapped == SPNIC_TX_TASK_WRAPPED) {
+ wqe_combo->bds_head = (struct spnic_sq_bufdesc *)
+ (void *)(txq->sq_head_addr);
+ } else {
+ wqe_combo->bds_head =
+ (struct spnic_sq_bufdesc *)(&wqe->extend_wqe.task);
+ }
+
+ if (wqe_info->wqebb_cnt > 1) {
+ wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE;
+ wqe_combo->task_type = SQ_WQE_TASKSECT_46BITS;
+ /* This section used as vlan insert, needs to clear */
+ wqe_combo->bds_head->rsvd = 0;
+ } else {
+ wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE;
+ }
+}
+
+void spnic_free_txq_mbufs(struct spnic_txq *txq)
+{
+ struct spnic_tx_info *tx_info = NULL;
+ u16 free_wqebbs;
+ u16 ci;
+
+ free_wqebbs = spnic_get_sq_free_wqebbs(txq) + 1;
+
+ while (free_wqebbs < txq->q_depth) {
+ ci = spnic_get_sq_local_ci(txq);
+
+ tx_info = &txq->tx_info[ci];
+
+ rte_pktmbuf_free(tx_info->mbuf);
+ spnic_update_sq_local_ci(txq, tx_info->wqebb_cnt);
+
+ free_wqebbs += tx_info->wqebb_cnt;
+ tx_info->mbuf = NULL;
+ }
+}
+
+void spnic_free_all_txq_mbufs(struct spnic_nic_dev *nic_dev)
+{
+ u16 qid;
+
+ for (qid = 0; qid < nic_dev->num_sqs; qid++)
+ spnic_free_txq_mbufs(nic_dev->txqs[qid]);
+}
+
+int spnic_start_all_sqs(struct rte_eth_dev *eth_dev)
+{
+ struct spnic_nic_dev *nic_dev = NULL;
+ int i;
+
+ nic_dev = SPNIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+
+ for (i = 0; i < nic_dev->num_rqs; i++)
+ eth_dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+
+ return 0;
+}
+static inline int spnic_xmit_mbuf_cleanup(struct spnic_txq *txq, u32 free_cnt)
+{
+ struct spnic_tx_info *tx_info = NULL;
+ struct rte_mbuf *mbuf = NULL;
+ struct rte_mbuf *mbuf_temp = NULL;
+ struct rte_mbuf *mbuf_free[SPNIC_MAX_TX_FREE_BULK];
+ int nb_free = 0;
+ int wqebb_cnt = 0;
+ u16 hw_ci, sw_ci, sq_mask;
+ u32 i;
+
+ hw_ci = spnic_get_sq_hw_ci(txq);
+ sw_ci = spnic_get_sq_local_ci(txq);
+ sq_mask = txq->q_mask;
+
+ for (i = 0; i < free_cnt; ++i) {
+ tx_info = &txq->tx_info[sw_ci];
+ if (hw_ci == sw_ci ||
+ (((hw_ci - sw_ci) & sq_mask) < tx_info->wqebb_cnt))
+ break;
+
+ sw_ci = (sw_ci + tx_info->wqebb_cnt) & sq_mask;
+
+ wqebb_cnt += tx_info->wqebb_cnt;
+ mbuf = tx_info->mbuf;
+
+ if (likely(mbuf->nb_segs == 1)) {
+ mbuf_temp = rte_pktmbuf_prefree_seg(mbuf);
+ tx_info->mbuf = NULL;
+
+ if (unlikely(mbuf_temp == NULL))
+ continue;
+
+ mbuf_free[nb_free++] = mbuf_temp;
+ if (unlikely(mbuf_temp->pool != mbuf_free[0]->pool ||
+ nb_free >= SPNIC_MAX_TX_FREE_BULK)) {
+ rte_mempool_put_bulk(mbuf_free[0]->pool,
+ (void **)mbuf_free, (nb_free - 1));
+ nb_free = 0;
+ mbuf_free[nb_free++] = mbuf_temp;
+ }
+ } else {
+ rte_pktmbuf_free(mbuf);
+ tx_info->mbuf = NULL;
+ }
+ }
+
+ if (nb_free > 0)
+ rte_mempool_put_bulk(mbuf_free[0]->pool, (void **)mbuf_free,
+ nb_free);
+
+ spnic_update_sq_local_ci(txq, wqebb_cnt);
+ return i;
+}
+
+static int spnic_tx_done_cleanup(void *txq, u32 free_cnt)
+{
+ struct spnic_txq *tx_queue = txq;
+ u32 try_free_cnt = !free_cnt ? tx_queue->q_depth : free_cnt;
+
+ return spnic_xmit_mbuf_cleanup(tx_queue, try_free_cnt);
+}
+int spnic_stop_sq(struct spnic_txq *txq)
+{
+ struct spnic_nic_dev *nic_dev = txq->nic_dev;
+ unsigned long timeout;
+ int err = -EFAULT;
+ int free_wqebbs;
+
+ timeout = msecs_to_jiffies(SPNIC_FLUSH_QUEUE_TIMEOUT) + jiffies;
+ do {
+ spnic_tx_done_cleanup(txq, 0);
+ free_wqebbs = spnic_get_sq_free_wqebbs(txq) + 1;
+ if (free_wqebbs == txq->q_depth) {
+ err = 0;
+ break;
+ }
+
+ rte_delay_us(1);
+ } while (time_before(jiffies, timeout));
+
+ if (err)
+ PMD_DRV_LOG(WARNING, "%s Wait sq empty timeout, queue_idx: %u, sw_ci: %u, "
+ "hw_ci: %u, sw_pi: %u, free_wqebbs: %u, q_depth:%u\n",
+ nic_dev->dev_name, txq->q_id,
+ spnic_get_sq_local_ci(txq),
+ spnic_get_sq_hw_ci(txq),
+ MASKED_QUEUE_IDX(txq, txq->prod_idx),
+ free_wqebbs, txq->q_depth);
+
+ return err;
+}
+
+/* Should stop transmitting any packets before calling this function */
+void spnic_flush_txqs(struct spnic_nic_dev *nic_dev)
+{
+ u16 qid;
+ int err;
+
+ for (qid = 0; qid < nic_dev->num_sqs; qid++) {
+ err = spnic_stop_sq(nic_dev->txqs[qid]);
+ if (err)
+ PMD_DRV_LOG(ERR, "Stop sq%d failed", qid);
+ }
+}
@@ -4,6 +4,224 @@
#ifndef _SPNIC_TX_H_
#define _SPNIC_TX_H_
+/* Tx offload info */
+struct spnic_tx_offload_info {
+ u8 outer_l2_len;
+ u8 outer_l3_type;
+ u16 outer_l3_len;
+
+ u8 inner_l2_len;
+ u8 inner_l3_type;
+ u16 inner_l3_len;
+
+ u8 tunnel_length;
+ u8 tunnel_type;
+ u8 inner_l4_type;
+ u8 inner_l4_len;
+
+ u16 payload_offset;
+ u8 inner_l4_tcp_udp;
+ u8 rsvd0;
+};
+
+/* tx wqe ctx */
+struct spnic_wqe_info {
+ u8 around;
+ u8 cpy_mbuf_cnt;
+ u16 sge_cnt;
+
+ u8 offload;
+ u8 rsvd0;
+ u16 payload_offset;
+
+ u8 wrapped;
+ u8 owner;
+ u16 pi;
+
+ u16 wqebb_cnt;
+ u16 rsvd1;
+
+ u32 queue_info;
+};
+struct spnic_sq_wqe_desc {
+ u32 ctrl_len;
+ u32 queue_info;
+ u32 hi_addr;
+ u32 lo_addr;
+};
+
+/*
+ * Engine only pass first 12B TS field directly to uCode through metadata,
+ * vlan_offoad is used for hardware when vlan insert in tx
+ */
+struct spnic_sq_task {
+ u32 pkt_info0;
+ u32 ip_identify;
+ u32 pkt_info2; /* Rsvd for ipsec spi */
+ u32 vlan_offload;
+};
+
+struct spnic_sq_bufdesc {
+ u32 len; /* 31-bits Length, L2NIC only use length[17:0] */
+ u32 rsvd;
+ u32 hi_addr;
+ u32 lo_addr;
+};
+
+struct spnic_sq_compact_wqe {
+ struct spnic_sq_wqe_desc wqe_desc;
+};
+
+struct spnic_sq_extend_wqe {
+ struct spnic_sq_wqe_desc wqe_desc;
+ struct spnic_sq_task task;
+ struct spnic_sq_bufdesc buf_desc[0];
+};
+
+struct spnic_sq_wqe {
+ union {
+ struct spnic_sq_compact_wqe compact_wqe;
+ struct spnic_sq_extend_wqe extend_wqe;
+ };
+};
+
+/* Use section pointer to support non continuous wqe */
+struct spnic_sq_wqe_combo {
+ struct spnic_sq_wqe_desc *hdr;
+ struct spnic_sq_task *task;
+ struct spnic_sq_bufdesc *bds_head;
+ u32 wqe_type;
+ u32 task_type;
+};
+
+/* SQ ctrl info */
+enum sq_wqe_data_format {
+ SQ_NORMAL_WQE = 0,
+};
+
+enum sq_wqe_ec_type {
+ SQ_WQE_COMPACT_TYPE = 0,
+ SQ_WQE_EXTENDED_TYPE = 1,
+};
+
+#define COMPACT_WQE_MAX_CTRL_LEN 0x3FFF
+
+enum sq_wqe_tasksect_len_type {
+ SQ_WQE_TASKSECT_46BITS = 0,
+ SQ_WQE_TASKSECT_16BYTES = 1,
+};
+
+#define SQ_CTRL_BD0_LEN_SHIFT 0
+#define SQ_CTRL_RSVD_SHIFT 18
+#define SQ_CTRL_BUFDESC_NUM_SHIFT 19
+#define SQ_CTRL_TASKSECT_LEN_SHIFT 27
+#define SQ_CTRL_DATA_FORMAT_SHIFT 28
+#define SQ_CTRL_DIRECT_SHIFT 29
+#define SQ_CTRL_EXTENDED_SHIFT 30
+#define SQ_CTRL_OWNER_SHIFT 31
+
+#define SQ_CTRL_BD0_LEN_MASK 0x3FFFFU
+#define SQ_CTRL_RSVD_MASK 0x1U
+#define SQ_CTRL_BUFDESC_NUM_MASK 0xFFU
+#define SQ_CTRL_TASKSECT_LEN_MASK 0x1U
+#define SQ_CTRL_DATA_FORMAT_MASK 0x1U
+#define SQ_CTRL_DIRECT_MASK 0x1U
+#define SQ_CTRL_EXTENDED_MASK 0x1U
+#define SQ_CTRL_OWNER_MASK 0x1U
+
+#define SQ_CTRL_SET(val, member) (((u32)(val) & \
+ SQ_CTRL_##member##_MASK) << \
+ SQ_CTRL_##member##_SHIFT)
+
+#define SQ_CTRL_GET(val, member) (((val) >> SQ_CTRL_##member##_SHIFT) \
+ & SQ_CTRL_##member##_MASK)
+
+#define SQ_CTRL_CLEAR(val, member) ((val) & \
+ (~(SQ_CTRL_##member##_MASK << \
+ SQ_CTRL_##member##_SHIFT)))
+
+#define SQ_CTRL_QUEUE_INFO_PKT_TYPE_SHIFT 0
+#define SQ_CTRL_QUEUE_INFO_PLDOFF_SHIFT 2
+#define SQ_CTRL_QUEUE_INFO_UFO_SHIFT 10
+#define SQ_CTRL_QUEUE_INFO_TSO_SHIFT 11
+#define SQ_CTRL_QUEUE_INFO_TCPUDP_CS_SHIFT 12
+#define SQ_CTRL_QUEUE_INFO_MSS_SHIFT 13
+#define SQ_CTRL_QUEUE_INFO_SCTP_SHIFT 27
+#define SQ_CTRL_QUEUE_INFO_UC_SHIFT 28
+#define SQ_CTRL_QUEUE_INFO_PRI_SHIFT 29
+
+#define SQ_CTRL_QUEUE_INFO_PKT_TYPE_MASK 0x3U
+#define SQ_CTRL_QUEUE_INFO_PLDOFF_MASK 0xFFU
+#define SQ_CTRL_QUEUE_INFO_UFO_MASK 0x1U
+#define SQ_CTRL_QUEUE_INFO_TSO_MASK 0x1U
+#define SQ_CTRL_QUEUE_INFO_TCPUDP_CS_MASK 0x1U
+#define SQ_CTRL_QUEUE_INFO_MSS_MASK 0x3FFFU
+#define SQ_CTRL_QUEUE_INFO_SCTP_MASK 0x1U
+#define SQ_CTRL_QUEUE_INFO_UC_MASK 0x1U
+#define SQ_CTRL_QUEUE_INFO_PRI_MASK 0x7U
+
+#define SQ_CTRL_QUEUE_INFO_SET(val, member) \
+ (((u32)(val) & SQ_CTRL_QUEUE_INFO_##member##_MASK) \
+ << SQ_CTRL_QUEUE_INFO_##member##_SHIFT)
+
+#define SQ_CTRL_QUEUE_INFO_GET(val, member) \
+ (((val) >> SQ_CTRL_QUEUE_INFO_##member##_SHIFT) \
+ & SQ_CTRL_QUEUE_INFO_##member##_MASK)
+
+#define SQ_CTRL_QUEUE_INFO_CLEAR(val, member) \
+ ((val) & (~(SQ_CTRL_QUEUE_INFO_##member##_MASK << \
+ SQ_CTRL_QUEUE_INFO_##member##_SHIFT)))
+
+#define SQ_TASK_INFO0_TUNNEL_FLAG_SHIFT 19
+#define SQ_TASK_INFO0_ESP_NEXT_PROTO_SHIFT 22
+#define SQ_TASK_INFO0_INNER_L4_EN_SHIFT 24
+#define SQ_TASK_INFO0_INNER_L3_EN_SHIFT 25
+#define SQ_TASK_INFO0_INNER_L4_PSEUDO_SHIFT 26
+#define SQ_TASK_INFO0_OUT_L4_EN_SHIFT 27
+#define SQ_TASK_INFO0_OUT_L3_EN_SHIFT 28
+#define SQ_TASK_INFO0_OUT_L4_PSEUDO_SHIFT 29
+#define SQ_TASK_INFO0_ESP_OFFLOAD_SHIFT 30
+#define SQ_TASK_INFO0_IPSEC_PROTO_SHIFT 31
+
+#define SQ_TASK_INFO0_TUNNEL_FLAG_MASK 0x1U
+#define SQ_TASK_INFO0_ESP_NEXT_PROTO_MASK 0x3U
+#define SQ_TASK_INFO0_INNER_L4_EN_MASK 0x1U
+#define SQ_TASK_INFO0_INNER_L3_EN_MASK 0x1U
+#define SQ_TASK_INFO0_INNER_L4_PSEUDO_MASK 0x1U
+#define SQ_TASK_INFO0_OUT_L4_EN_MASK 0x1U
+#define SQ_TASK_INFO0_OUT_L3_EN_MASK 0x1U
+#define SQ_TASK_INFO0_OUT_L4_PSEUDO_MASK 0x1U
+#define SQ_TASK_INFO0_ESP_OFFLOAD_MASK 0x1U
+#define SQ_TASK_INFO0_IPSEC_PROTO_MASK 0x1U
+
+#define SQ_TASK_INFO0_SET(val, member) \
+ (((u32)(val) & SQ_TASK_INFO0_##member##_MASK) << \
+ SQ_TASK_INFO0_##member##_SHIFT)
+#define SQ_TASK_INFO0_GET(val, member) \
+ (((val) >> SQ_TASK_INFO0_##member##_SHIFT) & \
+ SQ_TASK_INFO0_##member##_MASK)
+
+#define SQ_TASK_INFO1_SET(val, member) \
+ (((val) & SQ_TASK_INFO1_##member##_MASK) << \
+ SQ_TASK_INFO1_##member##_SHIFT)
+#define SQ_TASK_INFO1_GET(val, member) \
+ (((val) >> SQ_TASK_INFO1_##member##_SHIFT) & \
+ SQ_TASK_INFO1_##member##_MASK)
+
+#define SQ_TASK_INFO3_VLAN_TAG_SHIFT 0
+#define SQ_TASK_INFO3_VLAN_TYPE_SHIFT 16
+#define SQ_TASK_INFO3_VLAN_TAG_VALID_SHIFT 19
+
+#define SQ_TASK_INFO3_VLAN_TAG_MASK 0xFFFFU
+#define SQ_TASK_INFO3_VLAN_TYPE_MASK 0x7U
+#define SQ_TASK_INFO3_VLAN_TAG_VALID_MASK 0x1U
+
+#define SQ_TASK_INFO3_SET(val, member) \
+ (((val) & SQ_TASK_INFO3_##member##_MASK) << \
+ SQ_TASK_INFO3_##member##_SHIFT)
+#define SQ_TASK_INFO3_GET(val, member) \
+ (((val) >> SQ_TASK_INFO3_##member##_SHIFT) & \
+ SQ_TASK_INFO3_##member##_MASK)
/* Txq info */
struct spnic_txq_stats {
@@ -59,4 +277,14 @@ struct spnic_txq {
struct spnic_txq_stats txq_stats;
} __rte_cache_aligned;
+void spnic_flush_txqs(struct spnic_nic_dev *nic_dev);
+
+void spnic_free_txq_mbufs(struct spnic_txq *txq);
+
+void spnic_free_all_txq_mbufs(struct spnic_nic_dev *nic_dev);
+
+u16 spnic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, u16 nb_pkts);
+
+int spnic_stop_sq(struct spnic_txq *txq);
+int spnic_start_all_sqs(struct rte_eth_dev *eth_dev);
#endif /* _SPNIC_TX_H_ */