From patchwork Tue Aug 16 15:49:30 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavan Nikhilesh Bhagavatula X-Patchwork-Id: 115177 X-Patchwork-Delegate: jerinj@marvell.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 3C45AA00C3; Tue, 16 Aug 2022 17:49:48 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id D70B840691; Tue, 16 Aug 2022 17:49:47 +0200 (CEST) Received: from mx0b-0016f401.pphosted.com (mx0a-0016f401.pphosted.com [67.231.148.174]) by mails.dpdk.org (Postfix) with ESMTP id 30E1C40150 for ; Tue, 16 Aug 2022 17:49:46 +0200 (CEST) Received: from pps.filterd (m0045849.ppops.net [127.0.0.1]) by mx0a-0016f401.pphosted.com (8.17.1.5/8.17.1.5) with ESMTP id 27GDv9Ew003637; Tue, 16 Aug 2022 08:49:45 -0700 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h=from : to : cc : subject : date : message-id : mime-version : content-transfer-encoding : content-type; s=pfpt0220; bh=kGkPcBcIjGLpd8qU+xgIctAL53GuKBSBoEKaMzTkcB4=; b=NQiSL93d/HeueW/5U7zrGOi3ioV1cLHxGN+ByllvZu+9TOEH7aKbtDPjhOpoNqklhdeH HGWZH7vsZAyVERRhJSuFF4VOVhCra2jZPX/7bDkqf6cy5vAZHyKcSyBuP+k+0u/ii9LK sp5RJJ7zzTrlbdKV75RL+rVStqptf51LRn6h6lRQRtCQhguG1WYLLj7G5je5mCfXPTo8 G6/wk9B29M0K4I0kDG6/2brtHFBwRVwYW0nC101x8Euf7lKjervYHOokRJqV4Of5U19y 6/LEFbgrQj0IG4UA40CgNQIOSFXkc8p/gX5HZnZnRxRf1m+UVK2Zdaq5dMwrgSRTEEEv wg== Received: from dc5-exch01.marvell.com ([199.233.59.181]) by mx0a-0016f401.pphosted.com (PPS) with ESMTPS id 3j04b9a9re-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT); Tue, 16 Aug 2022 08:49:45 -0700 Received: from DC5-EXCH01.marvell.com (10.69.176.38) by DC5-EXCH01.marvell.com (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.2; Tue, 16 Aug 2022 08:49:43 -0700 Received: from maili.marvell.com (10.69.176.80) by DC5-EXCH01.marvell.com (10.69.176.38) with Microsoft SMTP Server id 15.0.1497.2 via Frontend Transport; Tue, 16 Aug 2022 08:49:43 -0700 Received: from MININT-80QBFE8.corp.innovium.com (unknown [10.28.161.88]) by maili.marvell.com (Postfix) with ESMTP id 608A23F7053; Tue, 16 Aug 2022 08:49:39 -0700 (PDT) From: To: , Jay Jayatheerthan CC: , , , , , , , , , , , Pavan Nikhilesh Subject: [PATCH 1/3] eventdev: add element offset to event vector Date: Tue, 16 Aug 2022 21:19:30 +0530 Message-ID: <20220816154932.10168-1-pbhagavatula@marvell.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 X-Proofpoint-GUID: rhQT-uIFoYFpv_c5BTdqBRX2w-s7P-1J X-Proofpoint-ORIG-GUID: rhQT-uIFoYFpv_c5BTdqBRX2w-s7P-1J X-Proofpoint-Virus-Version: vendor=baseguard engine=ICAP:2.0.205,Aquarius:18.0.883,Hydra:6.0.517,FMLib:17.11.122.1 definitions=2022-08-16_08,2022-08-16_02,2022-06-22_01 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Pavan Nikhilesh Add ``elem_offset:12`` bit field event vector structure the bits are taken from ``rsvd:15``. The element offset defines the offset into the vector array at which valid elements start. The valid elements count will be equal to nb_elem - elem_offset. Update Rx/Tx adapter SW implementation to use elem_offset. Signed-off-by: Pavan Nikhilesh --- lib/eventdev/rte_event_eth_rx_adapter.c | 1 + lib/eventdev/rte_event_eth_tx_adapter.c | 7 ++++--- lib/eventdev/rte_eventdev.h | 8 ++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) -- 2.25.1 diff --git a/lib/eventdev/rte_event_eth_rx_adapter.c b/lib/eventdev/rte_event_eth_rx_adapter.c index bf8741d2ea..bd72f9b845 100644 --- a/lib/eventdev/rte_event_eth_rx_adapter.c +++ b/lib/eventdev/rte_event_eth_rx_adapter.c @@ -855,6 +855,7 @@ rxa_init_vector(struct event_eth_rx_adapter *rx_adapter, vec->vector_ev->port = vec->port; vec->vector_ev->queue = vec->queue; vec->vector_ev->attr_valid = true; + vec->vector_ev->elem_offset = 0; TAILQ_INSERT_TAIL(&rx_adapter->vector_list, vec, next); } diff --git a/lib/eventdev/rte_event_eth_tx_adapter.c b/lib/eventdev/rte_event_eth_tx_adapter.c index b4b37f1cae..da70883e0d 100644 --- a/lib/eventdev/rte_event_eth_tx_adapter.c +++ b/lib/eventdev/rte_event_eth_tx_adapter.c @@ -524,16 +524,17 @@ txa_process_event_vector(struct txa_service_data *txa, queue = vec->queue; tqi = txa_service_queue(txa, port, queue); if (unlikely(tqi == NULL || !tqi->added)) { - rte_pktmbuf_free_bulk(mbufs, vec->nb_elem); + rte_pktmbuf_free_bulk(&mbufs[vec->elem_offset], + vec->nb_elem - vec->elem_offset); rte_mempool_put(rte_mempool_from_obj(vec), vec); return 0; } - for (i = 0; i < vec->nb_elem; i++) { + for (i = vec->elem_offset; i < vec->nb_elem; i++) { nb_tx += rte_eth_tx_buffer(port, queue, tqi->tx_buf, mbufs[i]); } } else { - for (i = 0; i < vec->nb_elem; i++) { + for (i = vec->elem_offset; i < vec->nb_elem; i++) { port = mbufs[i]->port; queue = rte_event_eth_tx_adapter_txq_get(mbufs[i]); tqi = txa_service_queue(txa, port, queue); diff --git a/lib/eventdev/rte_eventdev.h b/lib/eventdev/rte_eventdev.h index 6a6f6ea4c1..b0698fe748 100644 --- a/lib/eventdev/rte_eventdev.h +++ b/lib/eventdev/rte_eventdev.h @@ -1060,8 +1060,12 @@ rte_event_dev_close(uint8_t dev_id); */ struct rte_event_vector { uint16_t nb_elem; - /**< Number of elements in this event vector. */ - uint16_t rsvd : 15; + /**< Total number of elements in this event vector. */ + uint16_t elem_offset : 12; + /**< Offset into the vector array where valid elements start from. + * The valid elements count would be nb_elem - elem_offset. + */ + uint16_t rsvd : 3; /**< Reserved for future use */ uint16_t attr_valid : 1; /**< Indicates that the below union attributes have valid information. From patchwork Tue Aug 16 15:49:31 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavan Nikhilesh Bhagavatula X-Patchwork-Id: 115178 X-Patchwork-Delegate: jerinj@marvell.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 4EAF1A00C3; Tue, 16 Aug 2022 17:49:55 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 21A7540DDA; Tue, 16 Aug 2022 17:49:53 +0200 (CEST) Received: from mx0b-0016f401.pphosted.com (mx0b-0016f401.pphosted.com [67.231.156.173]) by mails.dpdk.org (Postfix) with ESMTP id 87E4340150 for ; Tue, 16 Aug 2022 17:49:51 +0200 (CEST) Received: from pps.filterd (m0045851.ppops.net [127.0.0.1]) by mx0b-0016f401.pphosted.com (8.17.1.5/8.17.1.5) with ESMTP id 27G7a6hJ020292; Tue, 16 Aug 2022 08:49:51 -0700 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h=from : to : cc : subject : date : message-id : in-reply-to : references : mime-version : content-transfer-encoding : content-type; s=pfpt0220; bh=iDeSdQzoOlXTKOjEFxtx9hGnBvwEhNen/3ZcXB7ibcQ=; b=I+ze2FdflPAzL4ll0hLVuKAwPz2MAQK0EsR35JqC9+5qQymyPcTaN/fJiCMPiZW/nbyB Oiwrt3o5crH6xAqJObzZifzPERNWJaD5JA73BUGQK97VJoBLFdQYS40GWIDt59vGhUBr eFi3sBu3f3om178qF2TbbQsAom8WxWY2G6pEjh3W1EmhVnd1CAPRkuKNMhm83Wli3ROn AcW/A3wZx4ccAShZGoE+iY7RPQKg64JeQ2Rv5dkYHAGv0tBu00oFmSj9Eja5qlhDkxa4 CrcMVGRjeVX5pMu0p3YlUY3mt9MKWYeHbijgmM2t/52I0vIqJE7BnvfwR7Au6jDmW+78 kQ== Received: from dc5-exch01.marvell.com ([199.233.59.181]) by mx0b-0016f401.pphosted.com (PPS) with ESMTPS id 3hxbfkuypa-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT); Tue, 16 Aug 2022 08:49:50 -0700 Received: from DC5-EXCH02.marvell.com (10.69.176.39) by DC5-EXCH01.marvell.com (10.69.176.38) with Microsoft SMTP Server (TLS) id 15.0.1497.2; Tue, 16 Aug 2022 08:49:48 -0700 Received: from maili.marvell.com (10.69.176.80) by DC5-EXCH02.marvell.com (10.69.176.39) with Microsoft SMTP Server id 15.0.1497.18 via Frontend Transport; Tue, 16 Aug 2022 08:49:48 -0700 Received: from MININT-80QBFE8.corp.innovium.com (unknown [10.28.161.88]) by maili.marvell.com (Postfix) with ESMTP id 431913F7054; Tue, 16 Aug 2022 08:49:44 -0700 (PDT) From: To: , Sunil Kumar Kori , "Pavan Nikhilesh" CC: , , , , , , , , , , , Subject: [PATCH 2/3] examples: update event vector free routine Date: Tue, 16 Aug 2022 21:19:31 +0530 Message-ID: <20220816154932.10168-2-pbhagavatula@marvell.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220816154932.10168-1-pbhagavatula@marvell.com> References: <20220816154932.10168-1-pbhagavatula@marvell.com> MIME-Version: 1.0 X-Proofpoint-GUID: jGaOSK2gXFtWgi8P0Rk4Y6-HRSQ_OIjd X-Proofpoint-ORIG-GUID: jGaOSK2gXFtWgi8P0Rk4Y6-HRSQ_OIjd X-Proofpoint-Virus-Version: vendor=baseguard engine=ICAP:2.0.205,Aquarius:18.0.883,Hydra:6.0.517,FMLib:17.11.122.1 definitions=2022-08-16_08,2022-08-16_02,2022-06-22_01 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Pavan Nikhilesh Update event vector free routine to account for element offset while freeing elements. Signed-off-by: Pavan Nikhilesh --- app/test-eventdev/test_pipeline_common.c | 5 +++-- examples/l2fwd-event/l2fwd_common.c | 5 +++-- examples/l3fwd/l3fwd_event.c | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) -- 2.25.1 diff --git a/app/test-eventdev/test_pipeline_common.c b/app/test-eventdev/test_pipeline_common.c index 4f40d37659..4404c665f5 100644 --- a/app/test-eventdev/test_pipeline_common.c +++ b/app/test-eventdev/test_pipeline_common.c @@ -673,8 +673,9 @@ pipeline_vector_array_free(struct rte_event events[], uint16_t num) uint16_t i; for (i = 0; i < num; i++) { - rte_pktmbuf_free_bulk(events[i].vec->mbufs, - events[i].vec->nb_elem); + rte_pktmbuf_free_bulk( + &events[i].vec->mbufs[events[i].vec->elem_offset], + events[i].vec->nb_elem - events[i].vec->elem_offset); rte_mempool_put(rte_mempool_from_obj(events[i].vec), events[i].vec); } diff --git a/examples/l2fwd-event/l2fwd_common.c b/examples/l2fwd-event/l2fwd_common.c index 41a0d3f22f..5af7fa6127 100644 --- a/examples/l2fwd-event/l2fwd_common.c +++ b/examples/l2fwd-event/l2fwd_common.c @@ -121,8 +121,9 @@ l2fwd_event_vector_array_free(struct rte_event events[], uint16_t num) uint16_t i; for (i = 0; i < num; i++) { - rte_pktmbuf_free_bulk(events[i].vec->mbufs, - events[i].vec->nb_elem); + rte_pktmbuf_free_bulk( + &events[i].vec->mbufs[events[i].vec->elem_offset], + events[i].vec->nb_elem - events[i].vec->elem_offset); rte_mempool_put(rte_mempool_from_obj(events[i].vec), events[i].vec); } diff --git a/examples/l3fwd/l3fwd_event.c b/examples/l3fwd/l3fwd_event.c index 0b58475c85..984dad1ece 100644 --- a/examples/l3fwd/l3fwd_event.c +++ b/examples/l3fwd/l3fwd_event.c @@ -294,8 +294,9 @@ l3fwd_event_vector_array_free(struct rte_event events[], uint16_t num) uint16_t i; for (i = 0; i < num; i++) { - rte_pktmbuf_free_bulk(events[i].vec->mbufs, - events[i].vec->nb_elem); + rte_pktmbuf_free_bulk( + &events[i].vec->mbufs[events[i].vec->elem_offset], + events[i].vec->nb_elem - events[i].vec->elem_offset); rte_mempool_put(rte_mempool_from_obj(events[i].vec), events[i].vec); } From patchwork Tue Aug 16 15:49:32 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavan Nikhilesh Bhagavatula X-Patchwork-Id: 115179 X-Patchwork-Delegate: jerinj@marvell.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 2F930A00C3; Tue, 16 Aug 2022 17:50:01 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 17C87410D3; Tue, 16 Aug 2022 17:49:58 +0200 (CEST) Received: from mx0b-0016f401.pphosted.com (mx0a-0016f401.pphosted.com [67.231.148.174]) by mails.dpdk.org (Postfix) with ESMTP id 8193240150 for ; Tue, 16 Aug 2022 17:49:56 +0200 (CEST) Received: from pps.filterd (m0045849.ppops.net [127.0.0.1]) by mx0a-0016f401.pphosted.com (8.17.1.5/8.17.1.5) with ESMTP id 27GEBVLh003643; Tue, 16 Aug 2022 08:49:55 -0700 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=marvell.com; h=from : to : cc : subject : date : message-id : in-reply-to : references : mime-version : content-transfer-encoding : content-type; s=pfpt0220; bh=YOOirpylonNWBsZYSI3FbIAeXZfRo7mVsEPmB8QEiAU=; b=bQsjAowNQOPvLfkd2m8WDkB4zAByxRBNS2aL+lfWqnq3mawDiWXx+kSLsd4TJvyNRQz/ Qxu/EA5bRsuIMycAkVGnelcQGhZqnNRSCQJFTLVWPHFHd61+BczQBoc0hpoBymPB+2hu T6inxHrzHW1bR0m2TaU8gyujcokdAiOAK8gtBRgFRgjNWSH2lapTR9B3hLw6IriFRwjr RSJDx5WAkTaDs10EKaVy2EQIyXSLk1sXxQo2m3rVdgFaBeMK3eezbb2S3nJdxyHbdJET e+qCMOCzCDd3dCQj5x/Lr48GSktAaeNDx6lh96SuQZ662UKoILQi7J9aDXf5QeO+rPcj vQ== Received: from dc5-exch02.marvell.com ([199.233.59.182]) by mx0a-0016f401.pphosted.com (PPS) with ESMTPS id 3j04b9a9sa-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-SHA384 bits=256 verify=NOT); Tue, 16 Aug 2022 08:49:55 -0700 Received: from DC5-EXCH01.marvell.com (10.69.176.38) by DC5-EXCH02.marvell.com (10.69.176.39) with Microsoft SMTP Server (TLS) id 15.0.1497.18; Tue, 16 Aug 2022 08:49:53 -0700 Received: from maili.marvell.com (10.69.176.80) by DC5-EXCH01.marvell.com (10.69.176.38) with Microsoft SMTP Server id 15.0.1497.2 via Frontend Transport; Tue, 16 Aug 2022 08:49:53 -0700 Received: from MININT-80QBFE8.corp.innovium.com (unknown [10.28.161.88]) by maili.marvell.com (Postfix) with ESMTP id 73D113F7053; Tue, 16 Aug 2022 08:49:49 -0700 (PDT) From: To: , Pavan Nikhilesh , "Shijith Thotton" CC: , , , , , , , , , , Subject: [PATCH 3/3] event/cnxk: update event vector Tx routine Date: Tue, 16 Aug 2022 21:19:32 +0530 Message-ID: <20220816154932.10168-3-pbhagavatula@marvell.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220816154932.10168-1-pbhagavatula@marvell.com> References: <20220816154932.10168-1-pbhagavatula@marvell.com> MIME-Version: 1.0 X-Proofpoint-GUID: SdHw90QSPf3T8XKi3Z9cVkwpuiC5VQj_ X-Proofpoint-ORIG-GUID: SdHw90QSPf3T8XKi3Z9cVkwpuiC5VQj_ X-Proofpoint-Virus-Version: vendor=baseguard engine=ICAP:2.0.205,Aquarius:18.0.883,Hydra:6.0.517,FMLib:17.11.122.1 definitions=2022-08-16_08,2022-08-16_02,2022-06-22_01 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Pavan Nikhilesh Update event vector transmit routine to honor elem_offset. Use ``rte_event_vector::elem_offset`` to report partial vector transmission to the application when there is not enough space in the SQ. Signed-off-by: Pavan Nikhilesh --- drivers/event/cnxk/cn10k_worker.h | 100 +++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 29 deletions(-) -- 2.25.1 diff --git a/drivers/event/cnxk/cn10k_worker.h b/drivers/event/cnxk/cn10k_worker.h index a71e076ff8..7d1fbe9c1b 100644 --- a/drivers/event/cnxk/cn10k_worker.h +++ b/drivers/event/cnxk/cn10k_worker.h @@ -516,7 +516,15 @@ cn10k_sso_txq_fc_wait(const struct cn10k_eth_txq *txq) ; } -static __rte_always_inline void +static __rte_always_inline int32_t +cn10k_sso_sq_depth(const struct cn10k_eth_txq *txq) +{ + return (txq->nb_sqb_bufs_adj - + __atomic_load_n((int16_t *)txq->fc_mem, __ATOMIC_RELAXED)) + << txq->sqes_per_sqb_log2; +} + +static __rte_always_inline uint16_t cn10k_sso_tx_one(struct cn10k_sso_hws *ws, struct rte_mbuf *m, uint64_t *cmd, uint16_t lmt_id, uintptr_t lmt_addr, uint8_t sched_type, const uint64_t *txq_data, const uint32_t flags) @@ -529,6 +537,9 @@ cn10k_sso_tx_one(struct cn10k_sso_hws *ws, struct rte_mbuf *m, uint64_t *cmd, bool sec; txq = cn10k_sso_hws_xtract_meta(m, txq_data); + if (cn10k_sso_sq_depth(txq) <= 0) + return 0; + cn10k_nix_tx_skeleton(txq, cmd, flags, 0); /* Perform header writes before barrier * for TSO @@ -566,21 +577,29 @@ cn10k_sso_tx_one(struct cn10k_sso_hws *ws, struct rte_mbuf *m, uint64_t *cmd, cn10k_sso_txq_fc_wait(txq); roc_lmt_submit_steorl(lmt_id, pa); + + if (flags & NIX_TX_OFFLOAD_MBUF_NOFF_F) { + if (ref_cnt > 1) + rte_io_wmb(); + } + return 1; } -static __rte_always_inline void +static __rte_always_inline uint16_t cn10k_sso_vwqe_split_tx(struct cn10k_sso_hws *ws, struct rte_mbuf **mbufs, uint16_t nb_mbufs, uint64_t *cmd, uint16_t lmt_id, uintptr_t lmt_addr, uint8_t sched_type, const uint64_t *txq_data, const uint32_t flags) { - uint16_t port[4], queue[4]; uint16_t i, j, pkts, scalar; + uint16_t port[4], queue[4]; struct cn10k_eth_txq *txq; + uint16_t done, cnt; + int32_t space; scalar = nb_mbufs & (NIX_DESCS_PER_LOOP - 1); pkts = RTE_ALIGN_FLOOR(nb_mbufs, NIX_DESCS_PER_LOOP); - + cnt = 0; for (i = 0; i < pkts; i += NIX_DESCS_PER_LOOP) { port[0] = mbufs[i]->port; port[1] = mbufs[i + 1]->port; @@ -594,27 +613,42 @@ cn10k_sso_vwqe_split_tx(struct cn10k_sso_hws *ws, struct rte_mbuf **mbufs, if (((port[0] ^ port[1]) & (port[2] ^ port[3])) || ((queue[0] ^ queue[1]) & (queue[2] ^ queue[3]))) { - for (j = 0; j < 4; j++) - cn10k_sso_tx_one(ws, mbufs[i + j], cmd, lmt_id, - lmt_addr, sched_type, txq_data, - flags); + for (j = 0; j < 4; j++) { + done = cn10k_sso_tx_one( + ws, mbufs[i + j], cmd, lmt_id, lmt_addr, + sched_type, txq_data, flags); + if (!done) + goto fail; + rte_io_wmb(); + cnt++; + } } else { txq = (struct cn10k_eth_txq *)(txq_data[(txq_data[port[0]] >> 48) + queue[0]] & (BIT_ULL(48) - 1)); - cn10k_nix_xmit_pkts_vector(txq, (uint64_t *)ws, - &mbufs[i], 4, cmd, - flags | NIX_TX_VWQE_F); + space = cn10k_sso_sq_depth(txq); + if (space < NIX_DESCS_PER_LOOP) + goto fail; + cn10k_nix_xmit_pkts_vector( + txq, (uint64_t *)ws, &mbufs[i], + NIX_DESCS_PER_LOOP, cmd, flags | NIX_TX_VWQE_F); + cnt += NIX_DESCS_PER_LOOP; } } mbufs += i; for (i = 0; i < scalar; i++) { - cn10k_sso_tx_one(ws, mbufs[i], cmd, lmt_id, lmt_addr, - sched_type, txq_data, flags); + done = cn10k_sso_tx_one(ws, mbufs[i], cmd, lmt_id, lmt_addr, + sched_type, txq_data, flags); + if (!done) + break; + rte_io_wmb(); + cnt++; } +fail: + return cnt; } static __rte_always_inline uint16_t @@ -633,7 +667,12 @@ cn10k_sso_hws_event_tx(struct cn10k_sso_hws *ws, struct rte_event *ev, if (ev->event_type & RTE_EVENT_TYPE_VECTOR) { struct rte_mbuf **mbufs = ev->vec->mbufs; uint64_t meta = *(uint64_t *)ev->vec; + uint16_t offset, nb_pkts; + int32_t space; + nb_pkts = meta & 0xFFFF; + offset = (meta >> 16) & 0xFFF; + nb_pkts -= offset; if (meta & BIT(31)) { txq = (struct cn10k_eth_txq *)(txq_data[(txq_data[meta >> 32] >> @@ -641,29 +680,32 @@ cn10k_sso_hws_event_tx(struct cn10k_sso_hws *ws, struct rte_event *ev, (meta >> 48)] & (BIT_ULL(48) - 1)); - cn10k_nix_xmit_pkts_vector(txq, (uint64_t *)ws, mbufs, - meta & 0xFFFF, cmd, + /* Transmit based on queue depth */ + space = cn10k_sso_sq_depth(txq); + if (space <= 0) + return 0; + nb_pkts = nb_pkts < space ? nb_pkts : (uint16_t)space; + cn10k_nix_xmit_pkts_vector(txq, (uint64_t *)ws, + mbufs + offset, nb_pkts, cmd, flags | NIX_TX_VWQE_F); } else { - cn10k_sso_vwqe_split_tx( - ws, mbufs, meta & 0xFFFF, cmd, lmt_id, lmt_addr, - ev->sched_type, txq_data, flags); + nb_pkts = cn10k_sso_vwqe_split_tx( + ws, mbufs + offset, nb_pkts, cmd, lmt_id, + lmt_addr, ev->sched_type, txq_data, flags); } - rte_mempool_put(rte_mempool_from_obj(ev->vec), ev->vec); + if (!((meta & 0xFFFF) - nb_pkts - offset)) + rte_mempool_put(rte_mempool_from_obj(ev->vec), ev->vec); + else + *(uint64_t *)ev->vec = (meta & ~0xFFF0000UL) | + ((uint32_t)nb_pkts + offset) + << 16; rte_prefetch0(ws); - return 1; + return !((meta & 0xFFFF) - nb_pkts - offset); } m = ev->mbuf; - txq = cn10k_sso_hws_xtract_meta(m, txq_data); - if (((txq->nb_sqb_bufs_adj - - __atomic_load_n((int16_t *)txq->fc_mem, __ATOMIC_RELAXED)) - << txq->sqes_per_sqb_log2) <= 0) - return 0; - cn10k_sso_tx_one(ws, m, cmd, lmt_id, lmt_addr, ev->sched_type, txq_data, - flags); - - return 1; + return cn10k_sso_tx_one(ws, m, cmd, lmt_id, lmt_addr, ev->sched_type, + txq_data, flags); } #define T(name, sz, flags) \