From patchwork Fri Dec 8 08:07:59 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Junfeng Guo X-Patchwork-Id: 134956 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id F0CB5436A6; Fri, 8 Dec 2023 09:10:48 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id DC4ED42FB1; Fri, 8 Dec 2023 09:10:48 +0100 (CET) Received: from mgamail.intel.com (mgamail.intel.com [198.175.65.12]) by mails.dpdk.org (Postfix) with ESMTP id 85AC242F04 for ; Fri, 8 Dec 2023 09:10:47 +0100 (CET) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1702023048; x=1733559048; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=AaXnDuAqXTMgMAxPblSiBnPDwKOUNwRJaTbNCen9Exk=; b=jy7CFbFLR7ctwkk2iX2MyPPS8Tc9tKZ+EG+J9WOKovNrQpmutHY8MRio kDCBBewSd2NckfcH1AAkRg5bGK3+ochoaPmkUFFyHAfOhGmwBrjuuylEp GBjknCoiQ/JSQUD7KK5dX3mSa/sUfv6jqsbHomqJ5yMn7qLVWK9eBQGc/ KXpOuY/9041s8n/pDyKAV06QNzWRXfIQzVtMO3HfY1w0FHDmcmdbOmwkm G6rd6o0xmrnnWrRD57CdinRyWF8ffnQ2h4upJ9gdbNzV/HvCPu/tydPr8 BFkgo2dpo2cCt94kUIbp1XDHKFVvy+zudsuJ64dKquUF/vpM2Ob7akPIq g==; X-IronPort-AV: E=McAfee;i="6600,9927,10917"; a="1456701" X-IronPort-AV: E=Sophos;i="6.04,260,1695711600"; d="scan'208";a="1456701" Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orvoesa104.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 08 Dec 2023 00:10:47 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10917"; a="1103485784" X-IronPort-AV: E=Sophos;i="6.04,260,1695711600"; d="scan'208";a="1103485784" Received: from dpdk-jf-ntb-one.sh.intel.com ([10.67.110.217]) by fmsmga005.fm.intel.com with ESMTP; 08 Dec 2023 00:08:54 -0800 From: Junfeng Guo To: jingjing.wu@intel.com Cc: dev@dpdk.org, omkar.maslekar@intel.com, Junfeng Guo Subject: [RFC v1] raw/ntb: add support for 6th Gen Intel Xeon Date: Fri, 8 Dec 2023 16:07:59 +0800 Message-Id: <20231208080759.3484405-1-junfeng.guo@intel.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20230915070816.893282-1-junfeng.guo@intel.com> References: <20230915070816.893282-1-junfeng.guo@intel.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add support for the 6th Gen Intel Xeon Scalable processors. Note that NTB devices within the 6th Gen Intel Xeon compliant to PCIe 5.0 spec. Signed-off-by: Junfeng Guo --- Depends-on: series-30460 \ ("raw/ntb: refactor to support NTB on 5th Gen Intel Xeon") --- doc/guides/rawdevs/ntb.rst | 1 + drivers/raw/ntb/ntb.c | 2 ++ drivers/raw/ntb/ntb.h | 1 + drivers/raw/ntb/ntb_hw_intel.c | 66 +++++++++++++++++----------------- drivers/raw/ntb/ntb_hw_intel.h | 40 ++++++++++++--------- usertools/dpdk-devbind.py | 2 +- 6 files changed, 61 insertions(+), 51 deletions(-) diff --git a/doc/guides/rawdevs/ntb.rst b/doc/guides/rawdevs/ntb.rst index ee452af4fd..e663b7a088 100644 --- a/doc/guides/rawdevs/ntb.rst +++ b/doc/guides/rawdevs/ntb.rst @@ -153,6 +153,7 @@ Limitation This PMD is only supported on Intel Xeon Platforms: +- 6th Generation Intel® Xeon® Scalable Processors. (NTB GEN5 device id: 0x0DB4) - 5th Generation Intel® Xeon® Scalable Processors. (NTB GEN4 device id: 0x347E) - 4th Generation Intel® Xeon® Scalable Processors. (NTB GEN4 device id: 0x347E) - 3rd Generation Intel® Xeon® Scalable Processors. (NTB GEN4 device id: 0x347E) diff --git a/drivers/raw/ntb/ntb.c b/drivers/raw/ntb/ntb.c index 2a163eff60..f1e48b877d 100644 --- a/drivers/raw/ntb/ntb.c +++ b/drivers/raw/ntb/ntb.c @@ -26,6 +26,7 @@ static const struct rte_pci_id pci_id_ntb_map[] = { { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_GEN3) }, { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_GEN4) }, + { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_GEN5) }, { .vendor_id = 0, /* sentinel */ }, }; @@ -1380,6 +1381,7 @@ ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev) switch (pci_dev->id.device_id) { case NTB_INTEL_DEV_ID_B2B_GEN3: case NTB_INTEL_DEV_ID_B2B_GEN4: + case NTB_INTEL_DEV_ID_B2B_GEN5: hw->ntb_ops = &intel_ntb_ops; break; default: diff --git a/drivers/raw/ntb/ntb.h b/drivers/raw/ntb/ntb.h index d2e6d566ed..adbc164b48 100644 --- a/drivers/raw/ntb/ntb.h +++ b/drivers/raw/ntb/ntb.h @@ -19,6 +19,7 @@ extern int ntb_logtype; /* Device IDs */ #define NTB_INTEL_DEV_ID_B2B_GEN3 0x201C #define NTB_INTEL_DEV_ID_B2B_GEN4 0x347E +#define NTB_INTEL_DEV_ID_B2B_GEN5 0x0DB4 /* Reserved to app to use. */ #define NTB_SPAD_USER "spad_user_" diff --git a/drivers/raw/ntb/ntb_hw_intel.c b/drivers/raw/ntb/ntb_hw_intel.c index 3e2094437f..76dce5e95f 100644 --- a/drivers/raw/ntb/ntb_hw_intel.c +++ b/drivers/raw/ntb/ntb_hw_intel.c @@ -98,7 +98,7 @@ intel_ntb4_check_ppd_for_ICX(struct ntb_hw *hw, uint32_t reg_val) } static int -intel_ntb4_check_ppd(struct ntb_hw *hw) +intel_ntb4_5_check_ppd(struct ntb_hw *hw) { uint8_t revision_id; uint32_t reg_val; @@ -112,7 +112,7 @@ intel_ntb4_check_ppd(struct ntb_hw *hw) return -EIO; } - reg_val = rte_read32(hw->hw_addr + XEON_NTB4_PPD1_OFFSET); + reg_val = rte_read32(hw->hw_addr + XEON_NTB4_5_PPD1_OFFSET); /* Distinguish HW platform (3rd Gen Xeon) via PCI Revision ID */ if (revision_id <= NTB_PCI_DEV_REVISION_ICX_MAX && @@ -120,8 +120,8 @@ intel_ntb4_check_ppd(struct ntb_hw *hw) return intel_ntb4_check_ppd_for_ICX(hw, reg_val); /* Check connection topo type. Only support B2B. */ - switch (reg_val & XEON_NTB4_PPD_CONN_MASK) { - case XEON_NTB4_PPD_CONN_B2B: + switch (reg_val & XEON_NTB4_5_PPD_CONN_MASK) { + case XEON_NTB4_5_PPD_CONN_B2B: NTB_LOG(INFO, "Topo B2B (back to back) is using."); break; default: @@ -130,12 +130,12 @@ intel_ntb4_check_ppd(struct ntb_hw *hw) } /* Check device config status. */ - switch (reg_val & XEON_NTB4_PPD_DEV_MASK) { - case XEON_NTB4_PPD_DEV_DSD: + switch (reg_val & XEON_NTB4_5_PPD_DEV_MASK) { + case XEON_NTB4_5_PPD_DEV_DSD: NTB_LOG(INFO, "DSD, Downstream Device."); hw->topo = NTB_TOPO_B2B_DSD; break; - case XEON_NTB4_PPD_DEV_USD: + case XEON_NTB4_5_PPD_DEV_USD: NTB_LOG(INFO, "USD, Upstream device."); hw->topo = NTB_TOPO_B2B_USD; break; @@ -161,9 +161,9 @@ intel_ntb_dev_init(const struct rte_rawdev *dev) if (is_gen3_ntb(hw->pci_dev)) /* PPD is in config space for NTB Gen3 */ ret = intel_ntb3_check_ppd(hw); - else if (is_gen4_ntb(hw->pci_dev)) - /* PPD is in MMIO for NTB Gen4 */ - ret = intel_ntb4_check_ppd(hw); + else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) + /* PPD is in MMIO for NTB Gen4/5 */ + ret = intel_ntb4_5_check_ppd(hw); else { NTB_LOG(ERR, "Cannot init device for unsupported device."); return -ENOTSUP; @@ -264,9 +264,9 @@ intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx, base &= ~0xf; limit = base + size; rte_write64(limit, limit_addr); - } else if (is_gen4_ntb(hw->pci_dev)) { + } else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) { /* Set translate base address index register */ - xlat_off = XEON_NTB4_IM1XBASEIDX_OFFSET + + xlat_off = XEON_NTB4_5_IM1XBASEIDX_OFFSET + mw_idx * XEON_XBASEIDX_INTERVAL; xlat_addr = hw->hw_addr + xlat_off; rte_write16(rte_log2_u64(size), xlat_addr); @@ -321,8 +321,8 @@ intel_ntb_get_link_status(const struct rte_rawdev *dev) NTB_LOG(ERR, "Unable to get link status."); return -EIO; } - } else if (is_gen4_ntb(hw->pci_dev)) { - reg_off = XEON_NTB4_LINK_STATUS_OFFSET; + } else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) { + reg_off = XEON_NTB4_5_LINK_STATUS_OFFSET; reg_val = rte_read16(hw->hw_addr + reg_off); } else { NTB_LOG(ERR, "Cannot get link status for unsupported device."); @@ -368,7 +368,7 @@ intel_ntb_gen3_set_link(const struct ntb_hw *hw, bool up) } static int -intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up) +intel_ntb_gen4_5_set_link(const struct ntb_hw *hw, bool up) { uint32_t ntb_ctrl, ppd0; uint16_t link_ctrl; @@ -380,20 +380,20 @@ intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up) ntb_ctrl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP; rte_write32(ntb_ctrl, reg_addr); - reg_addr = hw->hw_addr + XEON_NTB4_LINK_CTRL_OFFSET; + reg_addr = hw->hw_addr + XEON_NTB4_5_LINK_CTRL_OFFSET; link_ctrl = rte_read16(reg_addr); - link_ctrl &= ~XEON_NTB4_LINK_CTRL_LINK_DIS; + link_ctrl &= ~XEON_NTB4_5_LINK_CTRL_LINK_DIS; rte_write16(link_ctrl, reg_addr); /* start link training */ - reg_addr = hw->hw_addr + XEON_NTB4_PPD0_OFFSET; + reg_addr = hw->hw_addr + XEON_NTB4_5_PPD0_OFFSET; ppd0 = rte_read32(reg_addr); - ppd0 |= XEON_NTB4_PPD_LINKTRN; + ppd0 |= XEON_NTB4_5_PPD_LINKTRN; rte_write32(ppd0, reg_addr); /* make sure link training has started */ ppd0 = rte_read32(reg_addr); - if (!(ppd0 & XEON_NTB4_PPD_LINKTRN)) { + if (!(ppd0 & XEON_NTB4_5_PPD_LINKTRN)) { NTB_LOG(ERR, "Link is not training."); return -EINVAL; } @@ -404,9 +404,9 @@ intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up) ntb_ctrl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP); rte_write32(ntb_ctrl, reg_addr); - reg_addr = hw->hw_addr + XEON_NTB4_LINK_CTRL_OFFSET; + reg_addr = hw->hw_addr + XEON_NTB4_5_LINK_CTRL_OFFSET; link_ctrl = rte_read16(reg_addr); - link_ctrl |= XEON_NTB4_LINK_CTRL_LINK_DIS; + link_ctrl |= XEON_NTB4_5_LINK_CTRL_LINK_DIS; rte_write16(link_ctrl, reg_addr); } @@ -421,8 +421,8 @@ intel_ntb_set_link(const struct rte_rawdev *dev, bool up) if (is_gen3_ntb(hw->pci_dev)) ret = intel_ntb_gen3_set_link(hw, up); - else if (is_gen4_ntb(hw->pci_dev)) - ret = intel_ntb_gen4_set_link(hw, up); + else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) + ret = intel_ntb_gen4_5_set_link(hw, up); else { NTB_LOG(ERR, "Cannot set link for unsupported device."); ret = -ENOTSUP; @@ -447,8 +447,8 @@ intel_ntb_spad_read(const struct rte_rawdev *dev, int spad, bool peer) if (is_gen3_ntb(hw->pci_dev)) reg_off = peer ? XEON_NTB3_B2B_SPAD_OFFSET : XEON_IM_SPAD_OFFSET; - else if (is_gen4_ntb(hw->pci_dev)) - reg_off = peer ? XEON_NTB4_B2B_SPAD_OFFSET : + else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) + reg_off = peer ? XEON_NTB4_5_B2B_SPAD_OFFSET : XEON_IM_SPAD_OFFSET; else { NTB_LOG(ERR, "Cannot read spad for unsupported device."); @@ -477,8 +477,8 @@ intel_ntb_spad_write(const struct rte_rawdev *dev, int spad, if (is_gen3_ntb(hw->pci_dev)) reg_off = peer ? XEON_NTB3_B2B_SPAD_OFFSET : XEON_IM_SPAD_OFFSET; - else if (is_gen4_ntb(hw->pci_dev)) - reg_off = peer ? XEON_NTB4_B2B_SPAD_OFFSET : + else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) + reg_off = peer ? XEON_NTB4_5_B2B_SPAD_OFFSET : XEON_IM_SPAD_OFFSET; else { NTB_LOG(ERR, "Cannot write spad for unsupported device."); @@ -516,9 +516,9 @@ intel_ntb_db_clear(const struct rte_rawdev *dev, uint64_t db_bits) db_off = XEON_IM_INT_STATUS_OFFSET; db_addr = hw->hw_addr + db_off; - if (is_gen4_ntb(hw->pci_dev)) - rte_write16(XEON_NTB4_SLOTSTS_DLLSCS, - hw->hw_addr + XEON_NTB4_SLOTSTS); + if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) + rte_write16(XEON_NTB4_5_SLOTSTS_DLLSCS, + hw->hw_addr + XEON_NTB4_5_SLOTSTS); rte_write64(db_bits, db_addr); return 0; @@ -578,8 +578,8 @@ intel_ntb_vector_bind(const struct rte_rawdev *dev, uint8_t intr, uint8_t msix) /* Bind intr source to msix vector */ if (is_gen3_ntb(hw->pci_dev)) reg_off = XEON_NTB3_INTVEC_OFFSET; - else if (is_gen4_ntb(hw->pci_dev)) - reg_off = XEON_NTB4_INTVEC_OFFSET; + else if (is_gen4_ntb(hw->pci_dev) || is_gen5_ntb(hw->pci_dev)) + reg_off = XEON_NTB4_5_INTVEC_OFFSET; else { NTB_LOG(ERR, "Cannot bind vectors for unsupported device."); return -ENOTSUP; diff --git a/drivers/raw/ntb/ntb_hw_intel.h b/drivers/raw/ntb/ntb_hw_intel.h index 5a45785b57..404dfcf2fc 100644 --- a/drivers/raw/ntb/ntb_hw_intel.h +++ b/drivers/raw/ntb/ntb_hw_intel.h @@ -42,11 +42,11 @@ #define XEON_CORERRSTS_OFFSET 0x0158 #define XEON_NTB3_LINK_STATUS_OFFSET 0x01a2 /* Link status and PPD are in MMIO but not config space for Gen4/5 NTB */ -#define XEON_NTB4_PPD0_OFFSET 0xb0d4 -#define XEON_NTB4_PPD1_OFFSET 0xb4c0 -#define XEON_NTB4_LINK_CTRL_OFFSET 0xb050 -#define XEON_NTB4_LINK_STATUS_OFFSET 0xb052 -#define XEON_NTB4_LINK_CTRL_LINK_DIS 0x0010 +#define XEON_NTB4_5_PPD0_OFFSET 0xb0d4 +#define XEON_NTB4_5_PPD1_OFFSET 0xb4c0 +#define XEON_NTB4_5_LINK_CTRL_OFFSET 0xb050 +#define XEON_NTB4_5_LINK_STATUS_OFFSET 0xb052 +#define XEON_NTB4_5_LINK_CTRL_LINK_DIS 0x0010 #define XEON_NTBCNTL_OFFSET 0x0000 @@ -56,16 +56,16 @@ #define XEON_IMBAR2XBASE_OFFSET 0x0020 /* SBAR4XLAT */ #define XEON_IMBAR2XLMT_OFFSET 0x0028 /* SBAR4LMT */ #define XEON_XBASEIDX_INTERVAL 0x0002 -#define XEON_NTB4_IM1XBASEIDX_OFFSET 0x0074 -#define XEON_NTB4_IM2XBASEIDX_OFFSET 0x0076 +#define XEON_NTB4_5_IM1XBASEIDX_OFFSET 0x0074 +#define XEON_NTB4_5_IM2XBASEIDX_OFFSET 0x0076 #define XEON_IM_INT_STATUS_OFFSET 0x0040 #define XEON_IM_INT_DISABLE_OFFSET 0x0048 #define XEON_IM_SPAD_OFFSET 0x0080 /* SPAD */ #define XEON_NTB3_B2B_SPAD_OFFSET 0x0180 /* NTB GEN3 B2B SPAD */ -#define XEON_NTB4_B2B_SPAD_OFFSET 0x8080 /* NTB GEN4/5 B2B SPAD */ +#define XEON_NTB4_5_B2B_SPAD_OFFSET 0x8080 /* NTB GEN4/5 B2B SPAD */ #define XEON_USMEMMISS_OFFSET 0x0070 #define XEON_NTB3_INTVEC_OFFSET 0x00d0 -#define XEON_NTB4_INTVEC_OFFSET 0x0050 +#define XEON_NTB4_5_INTVEC_OFFSET 0x0050 #define XEON_IM_DOORBELL_OFFSET 0x0100 /* SDOORBELL0 */ #define XEON_EMBAR0XBASE_OFFSET 0x4008 /* B2B_XLAT */ #define XEON_EMBAR1XBASE_OFFSET 0x4010 /* PBAR2XLAT */ @@ -95,15 +95,15 @@ #define XEON_GEN3_NTB4_PPD_DEV_DSD 0x1000 #define XEON_GEN3_NTB4_PPD_DEV_USD 0x0000 -#define XEON_NTB4_PPD_CONN_MASK 0x0700 -#define XEON_NTB4_PPD_CONN_B2B 0x0200 -#define XEON_NTB4_PPD_DEV_MASK 0x4000 -#define XEON_NTB4_PPD_DEV_DSD 0x4000 -#define XEON_NTB4_PPD_DEV_USD 0x0000 +#define XEON_NTB4_5_PPD_CONN_MASK 0x0700 +#define XEON_NTB4_5_PPD_CONN_B2B 0x0200 +#define XEON_NTB4_5_PPD_DEV_MASK 0x4000 +#define XEON_NTB4_5_PPD_DEV_DSD 0x4000 +#define XEON_NTB4_5_PPD_DEV_USD 0x0000 -#define XEON_NTB4_PPD_LINKTRN 0x0008 -#define XEON_NTB4_SLOTSTS 0xb05a -#define XEON_NTB4_SLOTSTS_DLLSCS 0x100 +#define XEON_NTB4_5_PPD_LINKTRN 0x0008 +#define XEON_NTB4_5_SLOTSTS 0xb05a +#define XEON_NTB4_5_SLOTSTS_DLLSCS 0x100 #define XEON_MW_COUNT 2 @@ -129,4 +129,10 @@ is_gen4_ntb(const struct rte_pci_device *pci_dev) return pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_GEN4; } +static inline bool +is_gen5_ntb(const struct rte_pci_device *pci_dev) +{ + return pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_GEN5; +} + #endif /* _NTB_HW_INTEL_H_ */ diff --git a/usertools/dpdk-devbind.py b/usertools/dpdk-devbind.py index 230c81c5f3..6cfe2aa7ed 100755 --- a/usertools/dpdk-devbind.py +++ b/usertools/dpdk-devbind.py @@ -60,7 +60,7 @@ 'SVendor': None, 'SDevice': None} intel_idxd_spr = {'Class': '08', 'Vendor': '8086', 'Device': '0b25', 'SVendor': None, 'SDevice': None} -intel_ntb = {'Class': '06', 'Vendor': '8086', 'Device': '201c,347e', +intel_ntb = {'Class': '06', 'Vendor': '8086', 'Device': '201c,347e,0db4', 'SVendor': None, 'SDevice': None} cnxk_sso = {'Class': '08', 'Vendor': '177d', 'Device': 'a0f9,a0fa',