From patchwork Thu Jul 2 09:23:40 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Radu Nicolau X-Patchwork-Id: 72807 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id 19B73A0524; Thu, 2 Jul 2020 11:24:09 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 499911D724; Thu, 2 Jul 2020 11:24:05 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id 2AEE21D707 for ; Thu, 2 Jul 2020 11:24:02 +0200 (CEST) IronPort-SDR: PZsTg9aP+HUTgM6MjAcpUSvmIIsX5zQ6kLAzH9AvuD1hdcHMdp2r4gm+YXkh3K8xy8LAZ6NLp9 tpd/2UKEKsCw== X-IronPort-AV: E=McAfee;i="6000,8403,9669"; a="164898087" X-IronPort-AV: E=Sophos;i="5.75,303,1589266800"; d="scan'208";a="164898087" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 02 Jul 2020 02:24:02 -0700 IronPort-SDR: e0TkbV1gjnOtbcERGMgTm9ljI35i/tK2FkQfqPKRqwpHElsb10eBS0tdIO5El4r2J/GfYYXkz9 5dGqm41pARww== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.75,303,1589266800"; d="scan'208";a="321478849" Received: from silpixa00383879.ir.intel.com ([10.237.222.142]) by FMSMGA003.fm.intel.com with ESMTP; 02 Jul 2020 02:24:01 -0700 From: Radu Nicolau To: dev@dpdk.org Cc: beilei.xing@intel.com, jia.guo@intel.com, bruce.richardson@intel.com, konstantin.ananyev@intel.com, jerinjacobk@gmail.com, Radu Nicolau Date: Thu, 2 Jul 2020 10:23:40 +0100 Message-Id: <1593681821-22357-2-git-send-email-radu.nicolau@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1593681821-22357-1-git-send-email-radu.nicolau@intel.com> References: <1591870283-7776-1-git-send-email-radu.nicolau@intel.com> <1593681821-22357-1-git-send-email-radu.nicolau@intel.com> Subject: [dpdk-dev] [PATCH v4 1/2] eal: add WC store functions X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Add rte_write32_wc and rte_write32_wc_relaxed functions that implement 32bit stores using write combining memory protocol. Provided generic stubs and x86 implementation. Signed-off-by: Radu Nicolau Acked-by: Bruce Richardson --- v4: address feedback and include ack lib/librte_eal/include/generic/rte_io.h | 47 +++++++++++++++++++++++++++ lib/librte_eal/x86/include/rte_io.h | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/lib/librte_eal/include/generic/rte_io.h b/lib/librte_eal/include/generic/rte_io.h index da457f7..7391782 100644 --- a/lib/librte_eal/include/generic/rte_io.h +++ b/lib/librte_eal/include/generic/rte_io.h @@ -229,6 +229,39 @@ rte_write32(uint32_t value, volatile void *addr); static inline void rte_write64(uint64_t value, volatile void *addr); +/** + * Write a 32-bit value to I/O device memory address addr using write + * combining memory write protocol. Depending on the platform write combining + * may not be available and/or may be treated as a hint and the behavior may + * fallback to a regular store. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write32_wc(uint32_t value, volatile void *addr); + +/** + * Write a 32-bit value to I/O device memory address addr using write + * combining memory write protocol. Depending on the platform write combining + * may not be available and/or may be treated as a hint and the behavior may + * fallback to a regular store. + * + * The relaxed version does not have additional I/O memory barrier, useful in + * accessing the device registers of integrated controllers which implicitly + * strongly ordered with respect to memory access. + * + * @param value + * Value to write + * @param addr + * I/O memory address to write the value to + */ +static inline void +rte_write32_wc_relaxed(uint32_t value, volatile void *addr); + + #endif /* __DOXYGEN__ */ #ifndef RTE_OVERRIDE_IO_H @@ -345,6 +378,20 @@ rte_write64(uint64_t value, volatile void *addr) rte_write64_relaxed(value, addr); } +#ifndef RTE_NATIVE_WRITE32_WC +rte_write32_wc(uint32_t value, volatile void *addr) +{ + rte_write32(value, addr); +} + +static __rte_always_inline void +rte_write32_wc_relaxed(uint32_t value, volatile void *addr) +{ + rte_write32_relaxed(value, addr); +} +#endif /* RTE_NATIVE_WRITE32_WC */ + + #endif /* RTE_OVERRIDE_IO_H */ #endif /* _RTE_IO_H_ */ diff --git a/lib/librte_eal/x86/include/rte_io.h b/lib/librte_eal/x86/include/rte_io.h index 2db71b1..c95ed67 100644 --- a/lib/librte_eal/x86/include/rte_io.h +++ b/lib/librte_eal/x86/include/rte_io.h @@ -9,8 +9,64 @@ extern "C" { #endif +#include "rte_cpuflags.h" + +#define RTE_NATIVE_WRITE32_WC #include "generic/rte_io.h" +/** + * @internal + * MOVDIRI wrapper. + */ +static __rte_always_inline void +_rte_x86_movdiri(uint32_t value, volatile void *addr) +{ + asm volatile( + /* MOVDIRI */ + ".byte 0x40, 0x0f, 0x38, 0xf9, 0x02" + : + : "a" (value), "d" (addr)); +} + +static __rte_always_inline void +rte_write32_wc(uint32_t value, volatile void *addr) +{ + static int _x86_movdiri_flag = -1; + if (_x86_movdiri_flag == 1) { + rte_wmb(); + _rte_x86_movdiri(value, addr); + } else if (_x86_movdiri_flag == 0) { + rte_write32(value, addr); + } else { + _x86_movdiri_flag = + (rte_cpu_get_flag_enabled(RTE_CPUFLAG_MOVDIRI) > 0); + if (_x86_movdiri_flag == 1) { + rte_wmb(); + _rte_x86_movdiri(value, addr); + } else { + rte_write32(value, addr); + } + } +} + +static __rte_always_inline void +rte_write32_wc_relaxed(uint32_t value, volatile void *addr) +{ + static int _x86_movdiri_flag = -1; + if (_x86_movdiri_flag == 1) { + _rte_x86_movdiri(value, addr); + } else if (_x86_movdiri_flag == 0) { + rte_write32_relaxed(value, addr); + } else { + _x86_movdiri_flag = + (rte_cpu_get_flag_enabled(RTE_CPUFLAG_MOVDIRI) > 0); + if (_x86_movdiri_flag == 1) + _rte_x86_movdiri(value, addr); + else + rte_write32_relaxed(value, addr); + } +} + #ifdef __cplusplus } #endif From patchwork Thu Jul 2 09:23:41 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Radu Nicolau X-Patchwork-Id: 72808 X-Patchwork-Delegate: david.marchand@redhat.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from dpdk.org (dpdk.org [92.243.14.124]) by inbox.dpdk.org (Postfix) with ESMTP id E4482A0524; Thu, 2 Jul 2020 11:24:14 +0200 (CEST) Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 753A51D733; Thu, 2 Jul 2020 11:24:06 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id ED0231D718 for ; Thu, 2 Jul 2020 11:24:04 +0200 (CEST) IronPort-SDR: L1xFBrVT4sCWrp/JBphT4jZk86/VbkkfrsFnU5HWHg6Ad02n/VxcBTUyA5KQ6sVrnP2xeZIC7B SoSyfbL/DTzA== X-IronPort-AV: E=McAfee;i="6000,8403,9669"; a="164898106" X-IronPort-AV: E=Sophos;i="5.75,303,1589266800"; d="scan'208";a="164898106" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga003.fm.intel.com ([10.253.24.29]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 02 Jul 2020 02:24:04 -0700 IronPort-SDR: DILUYD/wJYKMqXMJp78SeOwK+NDNW11BpLgp3duCc59jaYzlzQiktpp8iqbruoCbqNn4NiIscw gojgNkYGgsRA== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.75,303,1589266800"; d="scan'208";a="321478856" Received: from silpixa00383879.ir.intel.com ([10.237.222.142]) by FMSMGA003.fm.intel.com with ESMTP; 02 Jul 2020 02:24:02 -0700 From: Radu Nicolau To: dev@dpdk.org Cc: beilei.xing@intel.com, jia.guo@intel.com, bruce.richardson@intel.com, konstantin.ananyev@intel.com, jerinjacobk@gmail.com, Radu Nicolau Date: Thu, 2 Jul 2020 10:23:41 +0100 Message-Id: <1593681821-22357-3-git-send-email-radu.nicolau@intel.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1593681821-22357-1-git-send-email-radu.nicolau@intel.com> References: <1591870283-7776-1-git-send-email-radu.nicolau@intel.com> <1593681821-22357-1-git-send-email-radu.nicolau@intel.com> Subject: [dpdk-dev] [PATCH v4 2/2] net/i40e: use WC store to update queue tail registers X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Performance improvement: use a write combining store instead of a regular mmio write to update queue tail registers. Signed-off-by: Radu Nicolau Acked-by: Bruce Richardson --- v4: include ack drivers/net/i40e/base/i40e_osdep.h | 6 ++++++ drivers/net/i40e/i40e_rxtx.c | 8 ++++---- drivers/net/i40e/i40e_rxtx_vec_avx2.c | 4 ++-- drivers/net/i40e/i40e_rxtx_vec_sse.c | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/net/i40e/base/i40e_osdep.h b/drivers/net/i40e/base/i40e_osdep.h index 58be396..df414fd 100644 --- a/drivers/net/i40e/base/i40e_osdep.h +++ b/drivers/net/i40e/base/i40e_osdep.h @@ -138,6 +138,12 @@ static inline uint32_t i40e_read_addr(volatile void *addr) #define I40E_PCI_REG_WRITE_RELAXED(reg, value) \ rte_write32_relaxed((rte_cpu_to_le_32(value)), reg) +#define I40E_PCI_REG_WC_WRITE(queue, reg, value) \ + rte_write32_wc((rte_cpu_to_le_32(value)), reg) +#define I40E_PCI_REG_WC_WRITE_RELAXED(queue, reg, value) \ + rte_write32_wc_relaxed((rte_cpu_to_le_32(value)), reg) + + #define I40E_WRITE_FLUSH(a) I40E_READ_REG(a, I40E_GLGEN_STAT) #define I40EVF_WRITE_FLUSH(a) I40E_READ_REG(a, I40E_VFGEN_RSTAT) diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c index 840b6f3..64e43ac 100644 --- a/drivers/net/i40e/i40e_rxtx.c +++ b/drivers/net/i40e/i40e_rxtx.c @@ -760,7 +760,7 @@ i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) if (nb_hold > rxq->rx_free_thresh) { rx_id = (uint16_t) ((rx_id == 0) ? (rxq->nb_rx_desc - 1) : (rx_id - 1)); - I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + I40E_PCI_REG_WC_WRITE(rxq, rxq->qrx_tail, rx_id); nb_hold = 0; } rxq->nb_rx_hold = nb_hold; @@ -938,7 +938,7 @@ i40e_recv_scattered_pkts(void *rx_queue, if (nb_hold > rxq->rx_free_thresh) { rx_id = (uint16_t)(rx_id == 0 ? (rxq->nb_rx_desc - 1) : (rx_id - 1)); - I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + I40E_PCI_REG_WC_WRITE(rxq, rxq->qrx_tail, rx_id); nb_hold = 0; } rxq->nb_rx_hold = nb_hold; @@ -1249,7 +1249,7 @@ i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) (unsigned) tx_id, (unsigned) nb_tx); rte_cio_wmb(); - I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id); + I40E_PCI_REG_WC_WRITE_RELAXED(txq, txq->qtx_tail, tx_id); txq->tx_tail = tx_id; return nb_tx; @@ -1400,7 +1400,7 @@ tx_xmit_pkts(struct i40e_tx_queue *txq, txq->tx_tail = 0; /* Update the tx tail register */ - I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + I40E_PCI_REG_WC_WRITE(txq, txq->qtx_tail, txq->tx_tail); return nb_pkts; } diff --git a/drivers/net/i40e/i40e_rxtx_vec_avx2.c b/drivers/net/i40e/i40e_rxtx_vec_avx2.c index 3bcef13..294c1c4 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_avx2.c +++ b/drivers/net/i40e/i40e_rxtx_vec_avx2.c @@ -134,7 +134,7 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); /* Update the tail pointer on the NIC */ - I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + I40E_PCI_REG_WC_WRITE(rxq, rxq->qrx_tail, rx_id); } #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC @@ -921,7 +921,7 @@ i40e_xmit_fixed_burst_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; - I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + I40E_PCI_REG_WC_WRITE(txq, txq->qtx_tail, txq->tx_tail); return nb_pkts; } diff --git a/drivers/net/i40e/i40e_rxtx_vec_sse.c b/drivers/net/i40e/i40e_rxtx_vec_sse.c index 6985183..a4635e0 100644 --- a/drivers/net/i40e/i40e_rxtx_vec_sse.c +++ b/drivers/net/i40e/i40e_rxtx_vec_sse.c @@ -86,7 +86,7 @@ i40e_rxq_rearm(struct i40e_rx_queue *rxq) (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1)); /* Update the tail pointer on the NIC */ - I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id); + I40E_PCI_REG_WC_WRITE(rxq, rxq->qrx_tail, rx_id); } #ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC @@ -733,7 +733,7 @@ i40e_xmit_fixed_burst_vec(void *tx_queue, struct rte_mbuf **tx_pkts, txq->tx_tail = tx_id; - I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail); + I40E_PCI_REG_WC_WRITE(txq, txq->qtx_tail, txq->tx_tail); return nb_pkts; }