From patchwork Tue Sep 27 02:47:54 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feifei Wang X-Patchwork-Id: 116922 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A480EA00C2; Tue, 27 Sep 2022 04:48:16 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9964F41145; Tue, 27 Sep 2022 04:48:15 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id DC71C41145 for ; Tue, 27 Sep 2022 04:48:13 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id E87E41063; Mon, 26 Sep 2022 19:48:19 -0700 (PDT) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.116]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 779553F73D; Mon, 26 Sep 2022 19:48:10 -0700 (PDT) From: Feifei Wang To: Thomas Monjalon , Ferruh Yigit , Andrew Rybchenko , Ray Kinsella Cc: dev@dpdk.org, nd@arm.com, Feifei Wang , Honnappa Nagarahalli , Ruifeng Wang Subject: [PATCH v2 1/3] ethdev: add API for direct rearm mode Date: Tue, 27 Sep 2022 10:47:54 +0800 Message-Id: <20220927024756.947272-2-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220927024756.947272-1-feifei.wang2@arm.com> References: <20220927024756.947272-1-feifei.wang2@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add API for enabling direct rearm mode and for mapping RX and TX queues. Currently, the API supports 1:1(txq : rxq) mapping. Furthermore, to avoid Rx load Tx data directly, add API called 'rte_eth_txq_data_get' to get Tx sw_ring and its information. Suggested-by: Honnappa Nagarahalli Suggested-by: Ruifeng Wang Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- lib/ethdev/ethdev_driver.h | 9 ++++ lib/ethdev/ethdev_private.c | 1 + lib/ethdev/rte_ethdev.c | 37 ++++++++++++++ lib/ethdev/rte_ethdev.h | 95 ++++++++++++++++++++++++++++++++++++ lib/ethdev/rte_ethdev_core.h | 5 ++ lib/ethdev/version.map | 4 ++ 6 files changed, 151 insertions(+) diff --git a/lib/ethdev/ethdev_driver.h b/lib/ethdev/ethdev_driver.h index 47a55a419e..14f52907c1 100644 --- a/lib/ethdev/ethdev_driver.h +++ b/lib/ethdev/ethdev_driver.h @@ -58,6 +58,8 @@ struct rte_eth_dev { eth_rx_descriptor_status_t rx_descriptor_status; /** Check the status of a Tx descriptor */ eth_tx_descriptor_status_t tx_descriptor_status; + /** Use Tx mbufs for Rx to rearm */ + eth_rx_direct_rearm_t rx_direct_rearm; /** * Device data that is shared between primary and secondary processes @@ -486,6 +488,11 @@ typedef int (*eth_rx_enable_intr_t)(struct rte_eth_dev *dev, typedef int (*eth_rx_disable_intr_t)(struct rte_eth_dev *dev, uint16_t rx_queue_id); +/**< @internal Get Tx information of a transmit queue of an Ethernet device. */ +typedef void (*eth_txq_data_get_t)(struct rte_eth_dev *dev, + uint16_t tx_queue_id, + struct rte_eth_txq_data *txq_data); + /** @internal Release memory resources allocated by given Rx/Tx queue. */ typedef void (*eth_queue_release_t)(struct rte_eth_dev *dev, uint16_t queue_id); @@ -1138,6 +1145,8 @@ struct eth_dev_ops { eth_rxq_info_get_t rxq_info_get; /** Retrieve Tx queue information */ eth_txq_info_get_t txq_info_get; + /** Get the address where Tx data is stored */ + eth_txq_data_get_t txq_data_get; eth_burst_mode_get_t rx_burst_mode_get; /**< Get Rx burst mode */ eth_burst_mode_get_t tx_burst_mode_get; /**< Get Tx burst mode */ eth_fw_version_get_t fw_version_get; /**< Get firmware version */ diff --git a/lib/ethdev/ethdev_private.c b/lib/ethdev/ethdev_private.c index 48090c879a..bfe16c7d77 100644 --- a/lib/ethdev/ethdev_private.c +++ b/lib/ethdev/ethdev_private.c @@ -276,6 +276,7 @@ eth_dev_fp_ops_setup(struct rte_eth_fp_ops *fpo, fpo->rx_queue_count = dev->rx_queue_count; fpo->rx_descriptor_status = dev->rx_descriptor_status; fpo->tx_descriptor_status = dev->tx_descriptor_status; + fpo->rx_direct_rearm = dev->rx_direct_rearm; fpo->rxq.data = dev->data->rx_queues; fpo->rxq.clbk = (void **)(uintptr_t)dev->post_rx_burst_cbs; diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c index 0c2c1088c0..0dccec2e4b 100644 --- a/lib/ethdev/rte_ethdev.c +++ b/lib/ethdev/rte_ethdev.c @@ -1648,6 +1648,43 @@ rte_eth_dev_is_removed(uint16_t port_id) return ret; } +int +rte_eth_tx_queue_data_get(uint16_t port_id, uint16_t queue_id, + struct rte_eth_txq_data *txq_data) +{ + struct rte_eth_dev *dev; + + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV); + dev = &rte_eth_devices[port_id]; + + if (queue_id >= dev->data->nb_tx_queues) { + RTE_ETHDEV_LOG(ERR, "Invalid Tx queue_id=%u\n", queue_id); + return -EINVAL; + } + + if (txq_data == NULL) { + RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u Tx queue %u data to NULL\n", + port_id, queue_id); + return -EINVAL; + } + + if (dev->data->tx_queues == NULL || + dev->data->tx_queues[queue_id] == NULL) { + RTE_ETHDEV_LOG(ERR, + "Tx queue %"PRIu16" of device with port_id=%" + PRIu16" has not been setup\n", + queue_id, port_id); + return -EINVAL; + } + + if (*dev->dev_ops->txq_data_get == NULL) + return -ENOTSUP; + + dev->dev_ops->txq_data_get(dev, queue_id, txq_data); + + return 0; +} + static int rte_eth_rx_queue_check_split(const struct rte_eth_rxseg_split *rx_seg, uint16_t n_seg, uint32_t *mbp_buf_size, diff --git a/lib/ethdev/rte_ethdev.h b/lib/ethdev/rte_ethdev.h index 2e783536c1..daf7f05d62 100644 --- a/lib/ethdev/rte_ethdev.h +++ b/lib/ethdev/rte_ethdev.h @@ -1949,6 +1949,23 @@ struct rte_eth_txq_info { uint8_t queue_state; /**< one of RTE_ETH_QUEUE_STATE_*. */ } __rte_cache_min_aligned; +/** + * @internal + * Structure used to hold pointers to internal ethdev Tx data. + * The main purpose is to load and store Tx queue data in direct rearm mode. + */ +struct rte_eth_txq_data { + uint64_t *offloads; + void *tx_sw_ring; + volatile void *tx_ring; + uint16_t *tx_next_dd; + uint16_t *nb_tx_free; + uint16_t nb_tx_desc; + uint16_t tx_rs_thresh; + uint16_t tx_free_thresh; +} __rte_cache_min_aligned; + + /* Generic Burst mode flag definition, values can be ORed. */ /** @@ -4718,6 +4735,27 @@ int rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id, int rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id, const struct rte_eth_rxtx_callback *user_cb); +/** + * Get the address which Tx data is stored. + * + * @param port_id + * The port identifier of the Ethernet device. + * @param queue_id + * The Tx queue on the Ethernet device for which information + * will be retrieved. + * @param txq_data + * A pointer to a structure of type *rte_eth_txq_data* to be filled. + * + * @return + * - 0: Success + * - -ENODEV: If *port_id* is invalid. + * - -ENOTSUP: routine is not supported by the device PMD. + * - -EINVAL: The queue_id is out of range. + */ +__rte_experimental +int rte_eth_tx_queue_data_get(uint16_t port_id, uint16_t queue_id, + struct rte_eth_txq_data *txq_data); + /** * Retrieve information about given port's Rx queue. * @@ -6209,6 +6247,63 @@ rte_eth_tx_buffer(uint16_t port_id, uint16_t queue_id, return rte_eth_tx_buffer_flush(port_id, queue_id, buffer); } +/** + * @warning + * @b EXPERIMENTAL: this API may change, or be removed, without prior notice + * + * Put Tx buffers into Rx sw-ring and rearm descs. + * + * @param port_id + * Port identifying the receive side. + * @param queue_id + * The index of the transmit queue identifying the receive side. + * The value must be in the range [0, nb_rx_queue - 1] previously supplied + * to rte_eth_dev_configure(). + * @param txq_data + * A pointer to a structure of type *rte_eth_txq_data* to be filled. + * @return + * The number of direct-rearmed buffers. + */ +__rte_experimental +static __rte_always_inline uint16_t +rte_eth_rx_direct_rearm(uint16_t port_id, uint16_t queue_id, + struct rte_eth_txq_data *txq_data) +{ + uint16_t nb_rearm; + struct rte_eth_fp_ops *p; + void *qd; + +#ifdef RTE_ETHDEV_DEBUG_RX + if (port_id >= RTE_MAX_ETHPORTS || + queue_id >= RTE_MAX_QUEUES_PER_PORT) { + RTE_ETHDEV_LOG(ERR, + "Invalid port_id=%u or queue_id=%u\n", + port_id, queue_id); + return 0; + } +#endif + + p = &rte_eth_fp_ops[port_id]; + qd = p->rxq.data[queue_id]; + +#ifdef RTE_ETHDEV_DEBUG_RX + RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0); + + if (qd == NULL) { + RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u for port_id=%u\n", + queue_id, port_id); + return 0; + } + + if (!p->rx_direct_rearm) + return -ENOTSUP; +#endif + + nb_rearm = p->rx_direct_rearm(qd, txq_data); + + return nb_rearm; +} + #ifdef __cplusplus } #endif diff --git a/lib/ethdev/rte_ethdev_core.h b/lib/ethdev/rte_ethdev_core.h index dcf8adab92..6a328e2481 100644 --- a/lib/ethdev/rte_ethdev_core.h +++ b/lib/ethdev/rte_ethdev_core.h @@ -56,6 +56,9 @@ typedef int (*eth_rx_descriptor_status_t)(void *rxq, uint16_t offset); /** @internal Check the status of a Tx descriptor */ typedef int (*eth_tx_descriptor_status_t)(void *txq, uint16_t offset); +/** @internal Put Tx buffers into Rx sw-ring and rearm descs */ +typedef uint16_t (*eth_rx_direct_rearm_t)(void *rxq, struct rte_eth_txq_data *txq_data); + /** * @internal * Structure used to hold opaque pointers to internal ethdev Rx/Tx @@ -90,6 +93,8 @@ struct rte_eth_fp_ops { eth_rx_queue_count_t rx_queue_count; /** Check the status of a Rx descriptor. */ eth_rx_descriptor_status_t rx_descriptor_status; + /** Put Tx buffers into Rx sw-ring and rearm descs */ + eth_rx_direct_rearm_t rx_direct_rearm; /** Rx queues data. */ struct rte_ethdev_qdata rxq; uintptr_t reserved1[3]; diff --git a/lib/ethdev/version.map b/lib/ethdev/version.map index 03f52fee91..69e4c376d3 100644 --- a/lib/ethdev/version.map +++ b/lib/ethdev/version.map @@ -285,6 +285,10 @@ EXPERIMENTAL { rte_mtr_color_in_protocol_priority_get; rte_mtr_color_in_protocol_set; rte_mtr_meter_vlan_table_update; + + # added in 22.11 + rte_eth_tx_queue_data_get; + rte_eth_rx_direct_rearm; }; INTERNAL { From patchwork Tue Sep 27 02:47:55 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feifei Wang X-Patchwork-Id: 116923 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8EC91A00C2; Tue, 27 Sep 2022 04:48:22 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 5930B427ED; Tue, 27 Sep 2022 04:48:18 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id B4A8C427EC for ; Tue, 27 Sep 2022 04:48:16 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D5B2E165C; Mon, 26 Sep 2022 19:48:22 -0700 (PDT) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.116]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id E245E3F73D; Mon, 26 Sep 2022 19:48:13 -0700 (PDT) From: Feifei Wang To: Yuying Zhang , Beilei Xing , Ruifeng Wang Cc: dev@dpdk.org, nd@arm.com, Feifei Wang , Honnappa Nagarahalli Subject: [PATCH v2 2/3] net/i40e: enable direct rearm mode Date: Tue, 27 Sep 2022 10:47:55 +0800 Message-Id: <20220927024756.947272-3-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220927024756.947272-1-feifei.wang2@arm.com> References: <20220927024756.947272-1-feifei.wang2@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org For i40e driver, enable direct re-arm mode. This patch supports the case of mapping Rx/Tx queues from the same single lcore. Suggested-by: Honnappa Nagarahalli Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- drivers/net/i40e/i40e_ethdev.c | 1 + drivers/net/i40e/i40e_ethdev.h | 2 + drivers/net/i40e/i40e_rxtx.c | 19 ++++++ drivers/net/i40e/i40e_rxtx.h | 2 + drivers/net/i40e/i40e_rxtx_vec_neon.c | 93 +++++++++++++++++++++++++++ 5 files changed, 117 insertions(+) diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c index 65e689df32..649ec06f31 100644 --- a/drivers/net/i40e/i40e_ethdev.c +++ b/drivers/net/i40e/i40e_ethdev.c @@ -497,6 +497,7 @@ static const struct eth_dev_ops i40e_eth_dev_ops = { .flow_ops_get = i40e_dev_flow_ops_get, .rxq_info_get = i40e_rxq_info_get, .txq_info_get = i40e_txq_info_get, + .txq_data_get = i40e_txq_data_get, .rx_burst_mode_get = i40e_rx_burst_mode_get, .tx_burst_mode_get = i40e_tx_burst_mode_get, .timesync_enable = i40e_timesync_enable, diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h index fe943a45ff..ee13730917 100644 --- a/drivers/net/i40e/i40e_ethdev.h +++ b/drivers/net/i40e/i40e_ethdev.h @@ -1352,6 +1352,8 @@ void i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo); void i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_txq_info *qinfo); +void i40e_txq_data_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_txq_data *txq_data); int i40e_rx_burst_mode_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_burst_mode *mode); int i40e_tx_burst_mode_get(struct rte_eth_dev *dev, uint16_t queue_id, diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c index 788ffb51c2..be92ccd38a 100644 --- a/drivers/net/i40e/i40e_rxtx.c +++ b/drivers/net/i40e/i40e_rxtx.c @@ -3197,6 +3197,24 @@ i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.offloads = txq->offloads; } +void +i40e_txq_data_get(struct rte_eth_dev *dev, uint16_t queue_id, + struct rte_eth_txq_data *txq_data) +{ + struct i40e_tx_queue *txq; + + txq = dev->data->tx_queues[queue_id]; + + txq_data->offloads = &txq->offloads; + txq_data->tx_sw_ring = txq->sw_ring; + txq_data->tx_ring = txq->tx_ring; + txq_data->tx_next_dd = &txq->tx_next_dd; + txq_data->nb_tx_free = &txq->nb_tx_free; + txq_data->nb_tx_desc = txq->nb_tx_desc; + txq_data->tx_rs_thresh = txq->tx_rs_thresh; + txq_data->tx_free_thresh = txq->tx_free_thresh; +} + #ifdef RTE_ARCH_X86 static inline bool get_avx_supported(bool request_avx512) @@ -3321,6 +3339,7 @@ i40e_set_rx_function(struct rte_eth_dev *dev) PMD_INIT_LOG(DEBUG, "Using Vector Rx (port %d).", dev->data->port_id); dev->rx_pkt_burst = i40e_recv_pkts_vec; + dev->rx_direct_rearm = i40e_direct_rearm_vec; } #endif /* RTE_ARCH_X86 */ } else if (!dev->data->scattered_rx && ad->rx_bulk_alloc_allowed) { diff --git a/drivers/net/i40e/i40e_rxtx.h b/drivers/net/i40e/i40e_rxtx.h index 5e6eecc501..7a8fa2d1e8 100644 --- a/drivers/net/i40e/i40e_rxtx.h +++ b/drivers/net/i40e/i40e_rxtx.h @@ -216,6 +216,8 @@ uint16_t i40e_simple_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); uint16_t i40e_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); +uint16_t i40e_direct_rearm_vec(void *rx_queue, + struct rte_eth_txq_data *txq_data); int i40e_tx_queue_init(struct i40e_tx_queue *txq); int i40e_rx_queue_init(struct i40e_rx_queue *rxq); void i40e_free_tx_resources(struct i40e_tx_queue *txq); diff --git a/drivers/net/i40e/i40e_rxtx_vec_neon.c b/drivers/net/i40e/i40e_rxtx_vec_neon.c index 12e6f1cbcb..84e0159e08 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_neon.c +++ b/drivers/net/i40e/i40e_rxtx_vec_neon.c @@ -762,3 +762,96 @@ i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) { return i40e_rx_vec_dev_conf_condition_check_default(dev); } + +uint16_t +i40e_direct_rearm_vec(void *rx_queue, struct rte_eth_txq_data *txq_data) +{ + struct i40e_rx_queue *rxq = rx_queue; + struct i40e_rx_entry *rxep; + struct i40e_tx_entry *txep; + volatile union i40e_rx_desc *rxdp; + volatile struct i40e_tx_desc *tx_ring; + struct rte_mbuf *m; + uint16_t rx_id; + uint64x2_t dma_addr; + uint64_t paddr; + uint16_t i, n; + uint16_t nb_rearm; + + if (rxq->rxrearm_nb > txq_data->tx_rs_thresh && + *txq_data->nb_tx_free < txq_data->tx_free_thresh) { + tx_ring = txq_data->tx_ring; + /* check DD bits on threshold descriptor */ + if ((tx_ring[*txq_data->tx_next_dd].cmd_type_offset_bsz & + rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) != + rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) { + return 0; + } + n = txq_data->tx_rs_thresh; + + /* first buffer to free from S/W ring is at index + * tx_next_dd - (tx_rs_thresh-1) + */ + txep = txq_data->tx_sw_ring; + txep += *txq_data->tx_next_dd - (txq_data->tx_rs_thresh - 1); + rxep = &rxq->sw_ring[rxq->rxrearm_start]; + rxdp = rxq->rx_ring + rxq->rxrearm_start; + + if (*txq_data->offloads & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { + /* directly put mbufs from Tx to Rx, + * and initialize the mbufs in vector + */ + for (i = 0; i < n; i++, rxep++, txep++) { + rxep[0].mbuf = txep[0].mbuf; + + /* Initialize rxdp descs */ + m = txep[0].mbuf; + paddr = m->buf_iova + RTE_PKTMBUF_HEADROOM; + dma_addr = vdupq_n_u64(paddr); + /* flush desc with pa dma_addr */ + vst1q_u64((uint64_t *)&rxdp++->read, dma_addr); + } + } else { + for (i = 0, nb_rearm = 0; i < n; i++) { + m = rte_pktmbuf_prefree_seg(txep[i].mbuf); + if (m != NULL) { + rxep[i].mbuf = m; + + /* Initialize rxdp descs */ + paddr = m->buf_iova + RTE_PKTMBUF_HEADROOM; + dma_addr = vdupq_n_u64(paddr); + /* flush desc with pa dma_addr */ + vst1q_u64((uint64_t *)&rxdp++->read, dma_addr); + nb_rearm++; + } + } + n = nb_rearm; + } + + /* update counters for Tx */ + *txq_data->nb_tx_free = *txq_data->nb_tx_free + txq_data->tx_rs_thresh; + *txq_data->tx_next_dd = *txq_data->tx_next_dd + txq_data->tx_rs_thresh; + if (*txq_data->tx_next_dd >= txq_data->nb_tx_desc) + *txq_data->tx_next_dd = txq_data->tx_rs_thresh - 1; + + /* Update the descriptor initializer index */ + rxq->rxrearm_start += n; + rx_id = rxq->rxrearm_start - 1; + + if (unlikely(rxq->rxrearm_start >= rxq->nb_rx_desc)) { + rxq->rxrearm_start = rxq->rxrearm_start - rxq->nb_rx_desc; + if (!rxq->rxrearm_start) + rx_id = rxq->nb_rx_desc - 1; + else + rx_id = rxq->rxrearm_start - 1; + } + rxq->rxrearm_nb -= n; + + rte_io_wmb(); + /* Update the tail pointer on the NIC */ + I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rx_id); + return n; + } + + return 0; +} From patchwork Tue Sep 27 02:47:56 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Feifei Wang X-Patchwork-Id: 116924 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id BD5C8A00C2; Tue, 27 Sep 2022 04:48:27 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 01459427F9; Tue, 27 Sep 2022 04:48:20 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 248FB427F5 for ; Tue, 27 Sep 2022 04:48:19 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 43D50165C; Mon, 26 Sep 2022 19:48:25 -0700 (PDT) Received: from net-x86-dell-8268.shanghai.arm.com (net-x86-dell-8268.shanghai.arm.com [10.169.210.116]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id CC0A93F73D; Mon, 26 Sep 2022 19:48:16 -0700 (PDT) From: Feifei Wang To: Cc: dev@dpdk.org, nd@arm.com, Feifei Wang , Honnappa Nagarahalli , Ruifeng Wang Subject: [PATCH v2 3/3] examples/l3fwd: enable direct rearm mode Date: Tue, 27 Sep 2022 10:47:56 +0800 Message-Id: <20220927024756.947272-4-feifei.wang2@arm.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220927024756.947272-1-feifei.wang2@arm.com> References: <20220927024756.947272-1-feifei.wang2@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Enable direct rearm mode in l3fwd. Users can use parameters: '--direct-rearm=(rx_portid,rx_queueid,tx_portid,tx_queueid)' to enable direct rearm. Suggested-by: Honnappa Nagarahalli Signed-off-by: Feifei Wang Reviewed-by: Ruifeng Wang Reviewed-by: Honnappa Nagarahalli --- examples/l3fwd/l3fwd.h | 12 +++++ examples/l3fwd/l3fwd_lpm.c | 22 +++++++++ examples/l3fwd/main.c | 94 +++++++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/examples/l3fwd/l3fwd.h b/examples/l3fwd/l3fwd.h index 40b5f32a9e..db097e344c 100644 --- a/examples/l3fwd/l3fwd.h +++ b/examples/l3fwd/l3fwd.h @@ -57,6 +57,10 @@ #endif #define HASH_ENTRY_NUMBER_DEFAULT 16 +/* MAX number of direct rearm mapping entry */ +#define MAX_DIRECT_REARM_ENTRY_NUMBER 16 +#define MAX_DIRECT_REARM_QUEUE_PER_PORT 8 + struct parm_cfg { const char *rule_ipv4_name; const char *rule_ipv6_name; @@ -114,6 +118,14 @@ extern struct parm_cfg parm_config; extern struct acl_algorithms acl_alg[]; +/* Used in direct rearm mode */ +extern bool enabled_direct_rearm; +extern uint8_t direct_rearm_entry_number; +extern bool queue_enabled_direct_rearm[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; +extern uint16_t direct_rearm_map_tx_port[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; +extern uint16_t direct_rearm_map_tx_queue[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; +extern uint8_t direct_rearm_entry_idx[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; + /* Send burst of packets on an output interface */ static inline int send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port) diff --git a/examples/l3fwd/l3fwd_lpm.c b/examples/l3fwd/l3fwd_lpm.c index 22d7f61a42..973fe70aae 100644 --- a/examples/l3fwd/l3fwd_lpm.c +++ b/examples/l3fwd/l3fwd_lpm.c @@ -150,6 +150,8 @@ lpm_main_loop(__rte_unused void *dummy) int i, nb_rx; uint16_t portid; uint8_t queueid; + uint8_t idx; + struct rte_eth_txq_data txq_data[MAX_DIRECT_REARM_ENTRY_NUMBER]; struct lcore_conf *qconf; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -175,6 +177,20 @@ lpm_main_loop(__rte_unused void *dummy) lcore_id, portid, queueid); } + if (enabled_direct_rearm) { + uint16_t tx_portid; + uint8_t tx_queueid; + + for (i = 0; i < n_rx_q; i++) { + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + tx_portid = direct_rearm_map_tx_port[portid][queueid]; + tx_queueid = direct_rearm_map_tx_queue[portid][queueid]; + idx = direct_rearm_entry_idx[portid][queueid]; + rte_eth_tx_queue_data_get(tx_portid, tx_queueid, &(txq_data[idx])); + } + } + cur_tsc = rte_rdtsc(); prev_tsc = cur_tsc; @@ -205,6 +221,12 @@ lpm_main_loop(__rte_unused void *dummy) for (i = 0; i < n_rx_q; ++i) { portid = qconf->rx_queue_list[i].port_id; queueid = qconf->rx_queue_list[i].queue_id; + + if (queue_enabled_direct_rearm[portid][queueid]) { + idx = direct_rearm_entry_idx[portid][queueid]; + rte_eth_rx_direct_rearm(portid, queueid, &(txq_data[idx])); + } + nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, MAX_PKT_BURST); if (nb_rx == 0) diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c index e090328fcc..2e9c5c0dc4 100644 --- a/examples/l3fwd/main.c +++ b/examples/l3fwd/main.c @@ -91,6 +91,20 @@ uint32_t enabled_port_mask; int ipv6; /**< ipv6 is false by default. */ uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT; +/* Used for direct rearm mode */ +bool enabled_direct_rearm;/**< Flag to enable direct rearm mode. */ +uint8_t direct_rearm_entry_number; /**< Number of entry for direct rearm map. */ +/**< Direct rearm config parameters. */ +uint16_t direct_rearm_config[MAX_DIRECT_REARM_ENTRY_NUMBER][4]; +/**< Enable direct rearm flag for Rx queue . */ +bool queue_enabled_direct_rearm[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; +/**< Matrix for Rx queue mapping Tx port in direct rearm mode. */ +uint16_t direct_rearm_map_tx_port[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; +/**< Matrix for Rx queue mapping Tx queue in direct rearm mode. */ +uint16_t direct_rearm_map_tx_queue[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; +/**< Matrix for Rx queue mapping entry idx in direct rearm mode. */ +uint8_t direct_rearm_entry_idx[RTE_MAX_ETHPORTS][MAX_DIRECT_REARM_QUEUE_PER_PORT]; + struct lcore_conf lcore_conf[RTE_MAX_LCORE]; struct parm_cfg parm_config; @@ -403,6 +417,7 @@ print_usage(const char *prgname) " [--mode]" " [--eventq-sched]" " [--event-vector [--event-vector-size SIZE] [--event-vector-tmo NS]]" + " --direct-rearm (rx_port, rx_queue, tx_port, tx_queue)[,(rx_port, rx_queue, tx_port, tx_queue)]" " [-E]" " [-L]\n\n" @@ -436,6 +451,7 @@ print_usage(const char *prgname) " --event-vector: Enable event vectorization.\n" " --event-vector-size: Max vector size if event vectorization is enabled.\n" " --event-vector-tmo: Max timeout to form vector in nanoseconds if event vectorization is enabled\n" + " --direct-rearm (rx_port, rx_queue, tx_port, tx_queue): Put Tx queue sw-ring buffers into Rx queue\n" " -E : Enable exact match, legacy flag please use --lookup=em instead\n" " -L : Enable longest prefix match, legacy flag please use --lookup=lpm instead\n" " --rule_ipv4=FILE: Specify the ipv4 rules entries file.\n" @@ -670,6 +686,53 @@ parse_lookup(const char *optarg) return 0; } +static int +parse_direct_rearm(const char *q_arg) +{ + char s[256]; + const char *p, *p0 = q_arg; + char *end; + enum fieldnames { + FLD_RX_PORT = 0, + FLD_RX_QUEUE, + FLD_TX_PORT, + FLD_TX_QUEUE, + _NUM_FLD + }; + unsigned long int_fld[_NUM_FLD]; + char *str_fld[_NUM_FLD]; + int i; + unsigned int size; + + while ((p = strchr(p0, '(')) != NULL) { + ++p; + p0 = strchr(p, ')'); + if (p0 == NULL) + return -1; + + size = p0 - p; + if (size >= sizeof(s)) + return -1; + + snprintf(s, sizeof(s), "%.*s", size, p); + if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD) + return -1; + for (i = 0; i < _NUM_FLD; i++) { + errno = 0; + int_fld[i] = strtoul(str_fld[i], &end, 0); + if (errno != 0 || end == str_fld[i] || int_fld[i] > 255) + return -1; + } + + direct_rearm_config[direct_rearm_entry_number][0] = int_fld[FLD_RX_PORT]; + direct_rearm_config[direct_rearm_entry_number][1] = int_fld[FLD_RX_QUEUE]; + direct_rearm_config[direct_rearm_entry_number][2] = int_fld[FLD_TX_PORT]; + direct_rearm_config[direct_rearm_entry_number][3] = int_fld[FLD_TX_QUEUE]; + ++direct_rearm_entry_number; + } + return 0; +} + #define MAX_JUMBO_PKT_LEN 9600 static const char short_options[] = @@ -696,6 +759,7 @@ static const char short_options[] = #define CMD_LINE_OPT_ENABLE_VECTOR "event-vector" #define CMD_LINE_OPT_VECTOR_SIZE "event-vector-size" #define CMD_LINE_OPT_VECTOR_TMO_NS "event-vector-tmo" +#define CMD_LINE_OPT_DIRECT_REARM "direct-rearm" #define CMD_LINE_OPT_RULE_IPV4 "rule_ipv4" #define CMD_LINE_OPT_RULE_IPV6 "rule_ipv6" #define CMD_LINE_OPT_ALG "alg" @@ -725,7 +789,8 @@ enum { CMD_LINE_OPT_LOOKUP_NUM, CMD_LINE_OPT_ENABLE_VECTOR_NUM, CMD_LINE_OPT_VECTOR_SIZE_NUM, - CMD_LINE_OPT_VECTOR_TMO_NS_NUM + CMD_LINE_OPT_VECTOR_TMO_NS_NUM, + CMD_LINE_OPT_DIRECT_REARM_NUM }; static const struct option lgopts[] = { @@ -747,6 +812,7 @@ static const struct option lgopts[] = { {CMD_LINE_OPT_ENABLE_VECTOR, 0, 0, CMD_LINE_OPT_ENABLE_VECTOR_NUM}, {CMD_LINE_OPT_VECTOR_SIZE, 1, 0, CMD_LINE_OPT_VECTOR_SIZE_NUM}, {CMD_LINE_OPT_VECTOR_TMO_NS, 1, 0, CMD_LINE_OPT_VECTOR_TMO_NS_NUM}, + {CMD_LINE_OPT_DIRECT_REARM, 1, 0, CMD_LINE_OPT_DIRECT_REARM_NUM}, {CMD_LINE_OPT_RULE_IPV4, 1, 0, CMD_LINE_OPT_RULE_IPV4_NUM}, {CMD_LINE_OPT_RULE_IPV6, 1, 0, CMD_LINE_OPT_RULE_IPV6_NUM}, {CMD_LINE_OPT_ALG, 1, 0, CMD_LINE_OPT_ALG_NUM}, @@ -912,6 +978,15 @@ parse_args(int argc, char **argv) case CMD_LINE_OPT_VECTOR_TMO_NS_NUM: evt_rsrc->vector_tmo_ns = strtoull(optarg, NULL, 10); break; + case CMD_LINE_OPT_DIRECT_REARM_NUM: + enabled_direct_rearm = 1; + ret = parse_direct_rearm(optarg); + if (ret) { + fprintf(stderr, "Invalid direct rearm map\n"); + print_usage(prgname); + return -1; + } + break; case CMD_LINE_OPT_RULE_IPV4_NUM: l3fwd_set_rule_ipv4_name(optarg); break; @@ -1594,6 +1669,23 @@ main(int argc, char **argv) } } + if (enabled_direct_rearm) { + uint16_t rx_port, tx_port; + uint8_t rx_queue, tx_queue; + uint8_t m = 0; + while (m < direct_rearm_entry_number) { + rx_port = direct_rearm_config[m][0]; + rx_queue = direct_rearm_config[m][1]; + tx_port = direct_rearm_config[m][2]; + tx_queue = direct_rearm_config[m][3]; + queue_enabled_direct_rearm[rx_port][rx_queue] = 1; + direct_rearm_map_tx_port[rx_port][rx_queue] = tx_port; + direct_rearm_map_tx_queue[rx_port][rx_queue] = tx_queue; + direct_rearm_entry_idx[rx_port][rx_queue] = m; + m++; + } + } + check_all_ports_link_status(enabled_port_mask); ret = 0;