From patchwork Wed Sep 11 02:07:36 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: WanRenyong X-Patchwork-Id: 143932 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CF9A24595A; Wed, 11 Sep 2024 04:10:11 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 298B74325C; Wed, 11 Sep 2024 04:08:31 +0200 (CEST) Received: from lf-1-15.ptr.blmpb.com (lf-1-15.ptr.blmpb.com [103.149.242.15]) by mails.dpdk.org (Postfix) with ESMTP id C098843245 for ; Wed, 11 Sep 2024 04:08:25 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=feishu2403070942; d=yunsilicon.com; t=1726020495; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=zYJSQbWsVmMu1M8kdNOf8ltF7YODz4QcIYKXaJvglG4=; b=aeoACPGm/oPTKvxmAyCk+DjPFcelfn9ug7Xjjyeb2/IzRHYimzsXuOXjP7O8QNHZwxkxxT h9o8JZ+/5YScRm6Sbe26T2C6itXX/+DrQt5nDdIyrHYyTPZW+e3ViruDWHrUC5ng1qrF7s foCdtzU+67L2NVkPDynmoRCbcwibk3Do1o2YXIv1awbzPDpmVaoADGee5Ks9fXUHEj6kVY 74VSUahf4u0KcFkNRir//u19e7Qt6LcS/BE4h8SSSJHLARoCTpYH63Cz5x0d/7PxSAmOJY S5SuRSGxjStP9CpoSW7DYcS+/UlX2mC/PRnewuey4G//seFxjA2hndoNyNoDOw== From: "WanRenyong" Date: Wed, 11 Sep 2024 10:07:36 +0800 Message-Id: <20240911020740.3950704-16-wanry@yunsilicon.com> To: X-Original-From: WanRenyong X-Lms-Return-Path: Mime-Version: 1.0 Cc: , , "WanRenyong" , "Rong Qian" Subject: [PATCH v2 15/19] net/xsc: add ethdev Tx burst Received: from ubuntu-liun.yunsilicon.com ([58.34.192.114]) by smtp.feishu.cn with ESMTPS; Wed, 11 Sep 2024 10:08:14 +0800 X-Mailer: git-send-email 2.25.1 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Implement xsc PMD transmit function. Signed-off-by: WanRenyong Signed-off-by: Rong Qian --- doc/guides/nics/features/xsc.ini | 4 + drivers/net/xsc/xsc_rxtx.c | 231 ++++++++++++++++++++++++++++++- drivers/net/xsc/xsc_rxtx.h | 9 ++ 3 files changed, 242 insertions(+), 2 deletions(-) diff --git a/doc/guides/nics/features/xsc.ini b/doc/guides/nics/features/xsc.ini index bdeb7a984b..772c6418c4 100644 --- a/doc/guides/nics/features/xsc.ini +++ b/doc/guides/nics/features/xsc.ini @@ -7,6 +7,10 @@ RSS hash = Y RSS key update = Y RSS reta update = Y +L3 checksum offload = Y +L4 checksum offload = Y +Inner L3 checksum = Y +Inner L4 checksum = Y Linux = Y ARMv8 = Y x86-64 = Y diff --git a/drivers/net/xsc/xsc_rxtx.c b/drivers/net/xsc/xsc_rxtx.c index 28360e62ff..7a31cd428c 100644 --- a/drivers/net/xsc/xsc_rxtx.c +++ b/drivers/net/xsc/xsc_rxtx.c @@ -14,6 +14,8 @@ #define XSC_CQE_OWNER_HW 0x2 #define XSC_CQE_OWNER_SW 0x4 #define XSC_CQE_OWNER_ERR 0x8 +#define XSC_OPCODE_RAW 0x7 +#define XSC_TX_COMP_CQE_HANDLE_MAX 2 #define XSC_MAX_RX_BURST_MBUFS 64 @@ -201,9 +203,234 @@ xsc_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) return nb_pkts; } +static __rte_always_inline void +xsc_tx_elts_free(struct xsc_txq_data *__rte_restrict txq, uint16_t tail) +{ + uint16_t elts_n = tail - txq->elts_tail; + uint32_t free_n; + + do { + free_n = txq->elts_s - (txq->elts_tail & txq->elts_m); + free_n = RTE_MIN(free_n, elts_n); + rte_pktmbuf_free_bulk(&txq->elts[txq->elts_tail & txq->elts_m], free_n); + txq->elts_tail += free_n; + elts_n -= free_n; + } while (elts_n > 0); +} + +static void +xsc_tx_cqes_handle(struct xsc_txq_data *__rte_restrict txq) +{ + uint32_t count = XSC_TX_COMP_CQE_HANDLE_MAX; + volatile struct xsc_cqe *last_cqe = NULL; + volatile struct xsc_cqe *cqe; + bool doorbell = false; + int ret; + uint16_t tail; + + do { + cqe = &txq->cqes[txq->cq_ci & txq->cqe_m]; + ret = check_cqe_own(cqe, txq->cqe_n, txq->cq_ci); + if (unlikely(ret != XSC_CQE_OWNER_SW)) { + if (likely(ret != XSC_CQE_OWNER_ERR)) + /* No new CQEs in completion queue. */ + break; + doorbell = true; + ++txq->cq_ci; + txq->cq_pi = txq->cq_ci; + last_cqe = NULL; + continue; + } + + doorbell = true; + ++txq->cq_ci; + last_cqe = cqe; + } while (--count > 0); + + if (likely(doorbell)) { + union xsc_cq_doorbell cq_db = { + .cq_data = 0 + }; + cq_db.next_cid = txq->cq_ci; + cq_db.cq_num = txq->cqn; + + /* Ring doorbell */ + rte_compiler_barrier(); + *txq->cq_db = rte_cpu_to_le_32(cq_db.cq_data); + + /* Release completed elts */ + if (likely(last_cqe != NULL)) { + txq->wqe_pi = rte_le_to_cpu_16(last_cqe->wqe_id) >> txq->wqe_ds_n; + tail = txq->fcqs[(txq->cq_ci - 1) & txq->cqe_m]; + if (likely(tail != txq->elts_tail)) + xsc_tx_elts_free(txq, tail); + } + } +} + +static __rte_always_inline void +xsc_tx_wqe_ctrl_seg_init(struct xsc_txq_data *__rte_restrict txq, + struct rte_mbuf *__rte_restrict mbuf, + struct xsc_wqe *__rte_restrict wqe) +{ + struct xsc_send_wqe_ctrl_seg *cs = &wqe->cseg; + int i = 0; + int ds_max = (1 << txq->wqe_ds_n) - 1; + + cs->msg_opcode = XSC_OPCODE_RAW; + cs->wqe_id = rte_cpu_to_le_16(txq->wqe_ci << txq->wqe_ds_n); + cs->has_pph = 0; + /* clear dseg's seg len */ + if (cs->ds_data_num > 1 && cs->ds_data_num <= ds_max) { + for (i = 1; i < cs->ds_data_num; i++) + wqe->dseg[i].seg_len = 0; + } + + cs->ds_data_num = mbuf->nb_segs; + if (mbuf->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) + cs->csum_en = 0x2; + else + cs->csum_en = 0; + + if (txq->tso_en == 1 && (mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG)) { + cs->has_pph = 0; + cs->so_type = 1; + cs->so_hdr_len = mbuf->l2_len + mbuf->l3_len + mbuf->l4_len; + cs->so_data_size = rte_cpu_to_le_16(mbuf->tso_segsz); + } + + cs->msg_len = rte_cpu_to_le_32(rte_pktmbuf_pkt_len(mbuf)); + if (unlikely(cs->msg_len == 0)) + cs->msg_len = rte_cpu_to_le_32(rte_pktmbuf_data_len(mbuf)); + + /* do not generate cqe for every pkts */ + cs->ce = 0; +} + +static __rte_always_inline void +xsc_tx_wqe_data_seg_init(struct rte_mbuf *mbuf, struct xsc_wqe *wqe) +{ + uint16_t i, nb_segs = mbuf->nb_segs; + uint32_t data_len; + rte_iova_t iova; + struct xsc_wqe_data_seg *dseg; + + for (i = 0; i < nb_segs; ++i) { + dseg = &wqe->dseg[i]; + iova = rte_pktmbuf_iova(mbuf); + data_len = rte_pktmbuf_data_len(mbuf); + + dseg->in_line = 0; + dseg->seg_len = rte_cpu_to_le_32(data_len); + dseg->lkey = 0; + dseg->va = rte_cpu_to_le_64(iova); + mbuf = mbuf->next; + } +} + +static __rte_always_inline struct xsc_wqe * +xsc_tx_wqes_fill(struct xsc_txq_data *__rte_restrict txq, + struct rte_mbuf **__rte_restrict pkts, + uint32_t pkts_n) +{ + uint32_t i; + struct xsc_wqe *wqe = NULL; + struct rte_mbuf *mbuf; + + for (i = 0; i < pkts_n; i++) { + mbuf = pkts[i]; + rte_prefetch0(mbuf); + wqe = (struct xsc_wqe *)((struct xsc_send_wqe_ctrl_seg *)txq->wqes + + (txq->wqe_ci & txq->wqe_m) * (1 << txq->wqe_ds_n)); + + /* init wqe ctrl seg */ + xsc_tx_wqe_ctrl_seg_init(txq, mbuf, wqe); + /* init wqe data segs */ + xsc_tx_wqe_data_seg_init(mbuf, wqe); + ++txq->wqe_ci; + } + + return wqe; +} + +static __rte_always_inline void +xsc_tx_doorbell_ring(volatile uint32_t *db, uint32_t index, + uint32_t qpn, uint16_t ds_n) +{ + union xsc_send_doorbell tx_db; + + rte_io_wmb(); + tx_db.next_pid = index << ds_n; + tx_db.qp_num = qpn; + *db = rte_cpu_to_le_32(tx_db.send_data); + + rte_wmb(); +} + +static __rte_always_inline void +xsc_tx_elts_store(struct xsc_txq_data *__rte_restrict txq, + struct rte_mbuf **__rte_restrict pkts, + uint32_t pkts_n) +{ + uint32_t part; + struct rte_mbuf **elts = (struct rte_mbuf **)txq->elts; + + part = txq->elts_s - (txq->elts_head & txq->elts_m); + rte_memcpy((void *)(elts + (txq->elts_head & txq->elts_m)), + (void *)pkts, + RTE_MIN(part, pkts_n) * sizeof(struct rte_mbuf *)); + + if (unlikely(part < pkts_n)) + rte_memcpy((void *)elts, (void *)(pkts + part), + (pkts_n - part) * sizeof(struct rte_mbuf *)); +} + uint16_t xsc_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { - return 0; -} + struct xsc_txq_data *txq = dpdk_txq; + uint32_t tx_n, remain_n = pkts_n; + uint16_t idx, elts_free, wqe_free; + uint16_t elts_head; + struct xsc_wqe *last_wqe; + if (unlikely(!pkts_n)) + return 0; +loop: + xsc_tx_cqes_handle(txq); + + elts_free = txq->elts_s - (uint16_t)(txq->elts_head - txq->elts_tail); + wqe_free = txq->wqe_s - (((txq->wqe_ci << txq->wqe_ds_n) - + (txq->wqe_pi << txq->wqe_ds_n)) >> txq->wqe_ds_n); + if (unlikely(elts_free == 0 || wqe_free == 0)) + goto exit; + + /* Fill in WQEs */ + tx_n = RTE_MIN(remain_n, wqe_free); + idx = pkts_n - remain_n; + last_wqe = xsc_tx_wqes_fill(txq, &pkts[idx], tx_n); + remain_n -= tx_n; + last_wqe->cseg.ce = 1; + + /* Update free-cqs, elts_comp */ + elts_head = txq->elts_head; + elts_head += tx_n; + if ((uint16_t)(elts_head - txq->elts_comp) > 0) { + txq->elts_comp = elts_head; + txq->fcqs[txq->cq_pi++ & txq->cqe_m] = elts_head; + } + + /* Ring tx doorbell */ + xsc_tx_doorbell_ring(txq->qp_db, txq->wqe_ci, + txq->qpn, txq->wqe_ds_n); + + xsc_tx_elts_store(txq, &pkts[idx], tx_n); + txq->elts_head += tx_n; + + if (remain_n > 0) + /* Try to process cqe, if wqe free is still 0, exit */ + goto loop; + +exit: + return pkts_n - remain_n; +} diff --git a/drivers/net/xsc/xsc_rxtx.h b/drivers/net/xsc/xsc_rxtx.h index 454d5c0378..ce6e47ad4c 100644 --- a/drivers/net/xsc/xsc_rxtx.h +++ b/drivers/net/xsc/xsc_rxtx.h @@ -166,6 +166,15 @@ struct __rte_cache_aligned xsc_rxq_data { uint32_t rss_hash:1; /* RSS hash enabled */ }; +union xsc_cq_doorbell { + struct { + uint32_t next_cid : 16; + uint32_t cq_num : 15; + uint32_t cq_sta : 1; + }; + uint32_t cq_data; +}; + union xsc_recv_doorbell { struct { uint32_t next_pid : 13;