From patchwork Thu May 30 17:48:08 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53916 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 5335D1B94A; Thu, 30 May 2019 19:48:35 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by dpdk.org (Postfix) with ESMTP id A54722B9A for ; Thu, 30 May 2019 19:48:28 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020076" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:26 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:08 -0700 Message-Id: <20190530174819.1160221-2-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 01/12] eal: Make rte_eal_using_phys_addrs work sooner X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This function only returned the correct answer after a call to initialize the memory subsystem. Make it work prior to that. Signed-off-by: Ben Walker Change-Id: I8f3c5128fbf5da884a956bbcc72c5a13564825d5 --- lib/librte_eal/linux/eal/eal_memory.c | 63 ++++++++++++--------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/lib/librte_eal/linux/eal/eal_memory.c b/lib/librte_eal/linux/eal/eal_memory.c index 416dad898..0c07bb946 100644 --- a/lib/librte_eal/linux/eal/eal_memory.c +++ b/lib/librte_eal/linux/eal/eal_memory.c @@ -66,34 +66,8 @@ * zone as well as a physical contiguous zone. */ -static bool phys_addrs_available = true; - #define RANDOMIZE_VA_SPACE_FILE "/proc/sys/kernel/randomize_va_space" -static void -test_phys_addrs_available(void) -{ - uint64_t tmp = 0; - phys_addr_t physaddr; - - if (!rte_eal_has_hugepages()) { - RTE_LOG(ERR, EAL, - "Started without hugepages support, physical addresses not available\n"); - phys_addrs_available = false; - return; - } - - physaddr = rte_mem_virt2phy(&tmp); - if (physaddr == RTE_BAD_PHYS_ADDR) { - if (rte_eal_iova_mode() == RTE_IOVA_PA) - RTE_LOG(ERR, EAL, - "Cannot obtain physical addresses: %s. " - "Only vfio will function.\n", - strerror(errno)); - phys_addrs_available = false; - } -} - /* * Get physical address of any mapped virtual address in the current process. */ @@ -107,7 +81,7 @@ rte_mem_virt2phy(const void *virtaddr) off_t offset; /* Cannot parse /proc/self/pagemap, no need to log errors everywhere */ - if (!phys_addrs_available) + if (!rte_eal_using_phys_addrs()) return RTE_BAD_IOVA; /* standard page size */ @@ -1336,8 +1310,6 @@ eal_legacy_hugepage_init(void) int nr_hugefiles, nr_hugepages = 0; void *addr; - test_phys_addrs_available(); - memset(used_hp, 0, sizeof(used_hp)); /* get pointer to global configuration */ @@ -1516,7 +1488,7 @@ eal_legacy_hugepage_init(void) continue; } - if (phys_addrs_available && + if (rte_eal_using_phys_addrs() && rte_eal_iova_mode() != RTE_IOVA_VA) { /* find physical addresses for each hugepage */ if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0) { @@ -1735,8 +1707,6 @@ eal_hugepage_init(void) uint64_t memory[RTE_MAX_NUMA_NODES]; int hp_sz_idx, socket_id; - test_phys_addrs_available(); - memset(used_hp, 0, sizeof(used_hp)); for (hp_sz_idx = 0; @@ -1879,8 +1849,6 @@ eal_legacy_hugepage_attach(void) "into secondary processes\n"); } - test_phys_addrs_available(); - fd_hugepage = open(eal_hugepage_data_path(), O_RDONLY); if (fd_hugepage < 0) { RTE_LOG(ERR, EAL, "Could not open %s\n", @@ -2020,7 +1988,32 @@ rte_eal_hugepage_attach(void) int rte_eal_using_phys_addrs(void) { - return phys_addrs_available; + static int using_phys_addrs = -1; + uint64_t tmp = 0; + phys_addr_t physaddr; + + if (using_phys_addrs != -1) + return using_phys_addrs; + + /* Set the default to 1 */ + using_phys_addrs = 1; + + if (!rte_eal_has_hugepages()) { + RTE_LOG(ERR, EAL, + "Started without hugepages support, physical addresses not available\n"); + using_phys_addrs = 0; + return using_phys_addrs; + } + + physaddr = rte_mem_virt2phy(&tmp); + if (physaddr == RTE_BAD_PHYS_ADDR) { + if (rte_eal_iova_mode() == RTE_IOVA_PA) + RTE_LOG(ERR, EAL, + "Cannot obtain physical addresses. Only vfio will function.\n"); + using_phys_addrs = 0; + } + + return using_phys_addrs; } static int __rte_unused From patchwork Thu May 30 17:48:09 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53918 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id C6D531B955; Thu, 30 May 2019 19:48:39 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by dpdk.org (Postfix) with ESMTP id 2920237A2 for ; Thu, 30 May 2019 19:48:29 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020082" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:26 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:09 -0700 Message-Id: <20190530174819.1160221-3-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 02/12] eal/pci: Inline several functions into rte_pci_get_iommu_class X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This is in preparation for future simplifications. The functions are simply inlined for now. Signed-off-by: Ben Walker Change-Id: I129992c9b44f4575a28cc05b78297e15b6be4249 --- drivers/bus/pci/linux/pci.c | 176 +++++++++++++++--------------------- 1 file changed, 71 insertions(+), 105 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index c99d523f0..d3177916a 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -497,86 +497,6 @@ rte_pci_scan(void) return -1; } -/* - * Is pci device bound to any kdrv - */ -static inline int -pci_one_device_is_bound(void) -{ - struct rte_pci_device *dev = NULL; - int ret = 0; - - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (dev->kdrv == RTE_KDRV_UNKNOWN || - dev->kdrv == RTE_KDRV_NONE) { - continue; - } else { - ret = 1; - break; - } - } - return ret; -} - -/* - * Any one of the device bound to uio - */ -static inline int -pci_one_device_bound_uio(void) -{ - struct rte_pci_device *dev = NULL; - struct rte_devargs *devargs; - int need_check; - - FOREACH_DEVICE_ON_PCIBUS(dev) { - devargs = dev->device.devargs; - - need_check = 0; - switch (rte_pci_bus.bus.conf.scan_mode) { - case RTE_BUS_SCAN_WHITELIST: - if (devargs && devargs->policy == RTE_DEV_WHITELISTED) - need_check = 1; - break; - case RTE_BUS_SCAN_UNDEFINED: - case RTE_BUS_SCAN_BLACKLIST: - if (devargs == NULL || - devargs->policy != RTE_DEV_BLACKLISTED) - need_check = 1; - break; - } - - if (!need_check) - continue; - - if (dev->kdrv == RTE_KDRV_IGB_UIO || - dev->kdrv == RTE_KDRV_UIO_GENERIC) { - return 1; - } - } - return 0; -} - -/* - * Any one of the device has iova as va - */ -static inline int -pci_one_device_has_iova_va(void) -{ - struct rte_pci_device *dev = NULL; - struct rte_pci_driver *drv = NULL; - - FOREACH_DRIVER_ON_PCIBUS(drv) { - if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (dev->kdrv == RTE_KDRV_VFIO && - rte_pci_match(drv, dev)) - return 1; - } - } - } - return 0; -} - #if defined(RTE_ARCH_X86) static bool pci_one_device_iommu_support_va(struct rte_pci_device *dev) @@ -641,14 +561,76 @@ pci_one_device_iommu_support_va(__rte_unused struct rte_pci_device *dev) #endif /* - * All devices IOMMUs support VA as IOVA + * Get iommu class of PCI devices on the bus. */ -static bool -pci_devices_iommu_support_va(void) +enum rte_iova_mode +rte_pci_get_iommu_class(void) { + bool is_bound = false; + bool is_vfio_noiommu_enabled = true; + bool has_iova_va = false; + bool is_bound_uio = false; + bool iommu_no_va = false; + bool break_out; + bool need_check; struct rte_pci_device *dev = NULL; struct rte_pci_driver *drv = NULL; + struct rte_devargs *devargs; + + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_UNKNOWN || + dev->kdrv == RTE_KDRV_NONE) { + continue; + } else { + is_bound = true; + break; + } + } + if (!is_bound) + return RTE_IOVA_DC; + FOREACH_DRIVER_ON_PCIBUS(drv) { + if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_VFIO && + rte_pci_match(drv, dev)) { + has_iova_va = true; + break; + } + } + + if (has_iova_va) + break; + } + } + + FOREACH_DEVICE_ON_PCIBUS(dev) { + devargs = dev->device.devargs; + + need_check = false; + switch (rte_pci_bus.bus.conf.scan_mode) { + case RTE_BUS_SCAN_WHITELIST: + if (devargs && devargs->policy == RTE_DEV_WHITELISTED) + need_check = true; + break; + case RTE_BUS_SCAN_UNDEFINED: + case RTE_BUS_SCAN_BLACKLIST: + if (devargs == NULL || + devargs->policy != RTE_DEV_BLACKLISTED) + need_check = true; + break; + } + + if (!need_check) + continue; + + if (dev->kdrv == RTE_KDRV_IGB_UIO || + dev->kdrv == RTE_KDRV_UIO_GENERIC) { + is_bound_uio = true; + } + } + + break_out = false; FOREACH_DRIVER_ON_PCIBUS(drv) { FOREACH_DEVICE_ON_PCIBUS(dev) { if (!rte_pci_match(drv, dev)) @@ -657,31 +639,15 @@ pci_devices_iommu_support_va(void) * just one PCI device needs to be checked out because * the IOMMU hardware is the same for all of them. */ - return pci_one_device_iommu_support_va(dev); + iommu_no_va = !pci_one_device_iommu_support_va(dev); + break_out = true; + break; } - } - return true; -} -/* - * Get iommu class of PCI devices on the bus. - */ -enum rte_iova_mode -rte_pci_get_iommu_class(void) -{ - bool is_bound; - bool is_vfio_noiommu_enabled = true; - bool has_iova_va; - bool is_bound_uio; - bool iommu_no_va; - - is_bound = pci_one_device_is_bound(); - if (!is_bound) - return RTE_IOVA_DC; + if (break_out) + break; + } - has_iova_va = pci_one_device_has_iova_va(); - is_bound_uio = pci_one_device_bound_uio(); - iommu_no_va = !pci_devices_iommu_support_va(); #ifdef VFIO_PRESENT is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ? true : false; From patchwork Thu May 30 17:48:10 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53920 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 5C93B1B96E; Thu, 30 May 2019 19:48:44 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by dpdk.org (Postfix) with ESMTP id 6F70D4CA6 for ; Thu, 30 May 2019 19:48:29 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020085" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:26 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:10 -0700 Message-Id: <20190530174819.1160221-4-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 03/12] eal/pci: Rework loops in rte_pci_get_iommu_class X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Make all of the loops first iterate over devices, then drivers. This is in preparation for combining them into a single loop. Signed-off-by: Ben Walker Change-Id: Ifb2bfcc60570a5d5a13481be3da0fc74bf00ef1f --- drivers/bus/pci/linux/pci.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index d3177916a..70815e4f0 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -589,10 +589,10 @@ rte_pci_get_iommu_class(void) if (!is_bound) return RTE_IOVA_DC; - FOREACH_DRIVER_ON_PCIBUS(drv) { - if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (dev->kdrv == RTE_KDRV_VFIO && + FOREACH_DEVICE_ON_PCIBUS(dev) { + if (dev->kdrv == RTE_KDRV_VFIO) { + FOREACH_DRIVER_ON_PCIBUS(drv) { + if (drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA && rte_pci_match(drv, dev)) { has_iova_va = true; break; @@ -631,8 +631,8 @@ rte_pci_get_iommu_class(void) } break_out = false; - FOREACH_DRIVER_ON_PCIBUS(drv) { - FOREACH_DEVICE_ON_PCIBUS(dev) { + FOREACH_DEVICE_ON_PCIBUS(dev) { + FOREACH_DRIVER_ON_PCIBUS(drv) { if (!rte_pci_match(drv, dev)) continue; /* From patchwork Thu May 30 17:48:11 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53923 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 376501B99E; Thu, 30 May 2019 19:48:49 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by dpdk.org (Postfix) with ESMTP id DC7EF1B944 for ; Thu, 30 May 2019 19:48:29 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020089" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:27 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:11 -0700 Message-Id: <20190530174819.1160221-5-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 04/12] eal/pci: Collapse two loops in rte_pci_get_iommu_class X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Two of these loops easily collapse into a single loop. This sets the stage for future simplifications. Signed-off-by: Ben Walker Change-Id: I3353f2e3585808cebff3f11805f96e4a1cc7fb3a --- drivers/bus/pci/linux/pci.c | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index 70815e4f0..b7a66d717 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -571,7 +571,6 @@ rte_pci_get_iommu_class(void) bool has_iova_va = false; bool is_bound_uio = false; bool iommu_no_va = false; - bool break_out; bool need_check; struct rte_pci_device *dev = NULL; struct rte_pci_driver *drv = NULL; @@ -592,8 +591,16 @@ rte_pci_get_iommu_class(void) FOREACH_DEVICE_ON_PCIBUS(dev) { if (dev->kdrv == RTE_KDRV_VFIO) { FOREACH_DRIVER_ON_PCIBUS(drv) { - if (drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA && - rte_pci_match(drv, dev)) { + if (!rte_pci_match(drv, dev)) + continue; + + /* + * just one PCI device needs to be checked out because + * the IOMMU hardware is the same for all of them. + */ + iommu_no_va = !pci_one_device_iommu_support_va(dev); + + if (drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { has_iova_va = true; break; } @@ -630,24 +637,6 @@ rte_pci_get_iommu_class(void) } } - break_out = false; - FOREACH_DEVICE_ON_PCIBUS(dev) { - FOREACH_DRIVER_ON_PCIBUS(drv) { - if (!rte_pci_match(drv, dev)) - continue; - /* - * just one PCI device needs to be checked out because - * the IOMMU hardware is the same for all of them. - */ - iommu_no_va = !pci_one_device_iommu_support_va(dev); - break_out = true; - break; - } - - if (break_out) - break; - } - #ifdef VFIO_PRESENT is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ? true : false; From patchwork Thu May 30 17:48:12 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53919 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 357431B95A; Thu, 30 May 2019 19:48:42 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by dpdk.org (Postfix) with ESMTP id 849714F91 for ; Thu, 30 May 2019 19:48:29 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020093" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:27 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:12 -0700 Message-Id: <20190530174819.1160221-6-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 05/12] eal/pci: Add function pci_ignore_device X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This performs a check for whether the device should be ignored due to whitelist or blacklist. This check eventually needs to apply to all of the other checks in rte_pci_get_iommu_class. Signed-off-by: Ben Walker Change-Id: I8e63e4c2e4199f34561ea1d911e13d6d74a47322 --- drivers/bus/pci/linux/pci.c | 44 +++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index b7a66d717..f269b6a64 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -560,6 +560,29 @@ pci_one_device_iommu_support_va(__rte_unused struct rte_pci_device *dev) } #endif +static bool +pci_ignore_device(struct rte_pci_device *dev) +{ + struct rte_devargs *devargs; + + devargs = dev->device.devargs; + + switch (rte_pci_bus.bus.conf.scan_mode) { + case RTE_BUS_SCAN_WHITELIST: + if (devargs && devargs->policy == RTE_DEV_WHITELISTED) + return false; + break; + case RTE_BUS_SCAN_UNDEFINED: + case RTE_BUS_SCAN_BLACKLIST: + if (devargs == NULL || + devargs->policy != RTE_DEV_BLACKLISTED) + return false; + break; + } + + return true; +} + /* * Get iommu class of PCI devices on the bus. */ @@ -571,10 +594,9 @@ rte_pci_get_iommu_class(void) bool has_iova_va = false; bool is_bound_uio = false; bool iommu_no_va = false; - bool need_check; struct rte_pci_device *dev = NULL; struct rte_pci_driver *drv = NULL; - struct rte_devargs *devargs; + FOREACH_DEVICE_ON_PCIBUS(dev) { if (dev->kdrv == RTE_KDRV_UNKNOWN || @@ -612,23 +634,7 @@ rte_pci_get_iommu_class(void) } FOREACH_DEVICE_ON_PCIBUS(dev) { - devargs = dev->device.devargs; - - need_check = false; - switch (rte_pci_bus.bus.conf.scan_mode) { - case RTE_BUS_SCAN_WHITELIST: - if (devargs && devargs->policy == RTE_DEV_WHITELISTED) - need_check = true; - break; - case RTE_BUS_SCAN_UNDEFINED: - case RTE_BUS_SCAN_BLACKLIST: - if (devargs == NULL || - devargs->policy != RTE_DEV_BLACKLISTED) - need_check = true; - break; - } - - if (!need_check) + if (pci_ignore_device(dev)) continue; if (dev->kdrv == RTE_KDRV_IGB_UIO || From patchwork Thu May 30 17:48:13 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53915 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 8589B4F91; Thu, 30 May 2019 19:48:32 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by dpdk.org (Postfix) with ESMTP id C04A6375B for ; Thu, 30 May 2019 19:48:28 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020097" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:27 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:13 -0700 Message-Id: <20190530174819.1160221-7-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 06/12] eal/pci: Correctly test whitelist/blacklist in rte_pci_get_iommu_class X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" All of the checks should respect the white and black lists. Signed-off-by: Ben Walker Change-Id: Ie66176bea49987d1fc0a03dbee2638d9dd6efbc5 --- drivers/bus/pci/linux/pci.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index f269b6a64..ebe62f140 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -597,8 +597,10 @@ rte_pci_get_iommu_class(void) struct rte_pci_device *dev = NULL; struct rte_pci_driver *drv = NULL; - FOREACH_DEVICE_ON_PCIBUS(dev) { + if (pci_ignore_device(dev)) + continue; + if (dev->kdrv == RTE_KDRV_UNKNOWN || dev->kdrv == RTE_KDRV_NONE) { continue; @@ -611,6 +613,9 @@ rte_pci_get_iommu_class(void) return RTE_IOVA_DC; FOREACH_DEVICE_ON_PCIBUS(dev) { + if (pci_ignore_device(dev)) + continue; + if (dev->kdrv == RTE_KDRV_VFIO) { FOREACH_DRIVER_ON_PCIBUS(drv) { if (!rte_pci_match(drv, dev)) From patchwork Thu May 30 17:48:14 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53921 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 509281B994; Thu, 30 May 2019 19:48:46 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by dpdk.org (Postfix) with ESMTP id 0D25B1B948 for ; Thu, 30 May 2019 19:48:29 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020100" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:27 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:14 -0700 Message-Id: <20190530174819.1160221-8-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 07/12] eal/pci: Reverse if check in rte_pci_get_iommu_class X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" It's simpler to reverse the if statement here, especially with an upcoming simplification. Signed-off-by: Ben Walker Change-Id: I6cff80231032304f3f865fdf38157554fad7fd07 --- drivers/bus/pci/linux/pci.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index ebe62f140..f678d2318 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -601,10 +601,8 @@ rte_pci_get_iommu_class(void) if (pci_ignore_device(dev)) continue; - if (dev->kdrv == RTE_KDRV_UNKNOWN || - dev->kdrv == RTE_KDRV_NONE) { - continue; - } else { + if (dev->kdrv != RTE_KDRV_UNKNOWN && + dev->kdrv != RTE_KDRV_NONE) { is_bound = true; break; } From patchwork Thu May 30 17:48:15 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53925 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 421661B9A9; Thu, 30 May 2019 19:48:52 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by dpdk.org (Postfix) with ESMTP id 3310537A2 for ; Thu, 30 May 2019 19:48:30 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020104" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:27 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:15 -0700 Message-Id: <20190530174819.1160221-9-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 08/12] eal/pci: Collapse loops in rte_pci_get_iommu_class X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The three loops can now be easily combined into one. This is slightly less efficient than before because it doesn't break out early. But that can be addressed later. Signed-off-by: Ben Walker Change-Id: Ic97155bb478dddbcbeaa6d51947684ffef219a52 --- drivers/bus/pci/linux/pci.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index f678d2318..11e2e4d1b 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -604,15 +604,7 @@ rte_pci_get_iommu_class(void) if (dev->kdrv != RTE_KDRV_UNKNOWN && dev->kdrv != RTE_KDRV_NONE) { is_bound = true; - break; } - } - if (!is_bound) - return RTE_IOVA_DC; - - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (pci_ignore_device(dev)) - continue; if (dev->kdrv == RTE_KDRV_VFIO) { FOREACH_DRIVER_ON_PCIBUS(drv) { @@ -630,15 +622,7 @@ rte_pci_get_iommu_class(void) break; } } - - if (has_iova_va) - break; } - } - - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (pci_ignore_device(dev)) - continue; if (dev->kdrv == RTE_KDRV_IGB_UIO || dev->kdrv == RTE_KDRV_UIO_GENERIC) { @@ -646,6 +630,9 @@ rte_pci_get_iommu_class(void) } } + if (!is_bound) + return RTE_IOVA_DC; + #ifdef VFIO_PRESENT is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ? true : false; From patchwork Thu May 30 17:48:16 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53922 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A77C91B999; Thu, 30 May 2019 19:48:47 +0200 (CEST) Received: from mga07.intel.com (mga07.intel.com [134.134.136.100]) by dpdk.org (Postfix) with ESMTP id 0BBBE378B for ; Thu, 30 May 2019 19:48:29 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:27 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020107" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:27 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:16 -0700 Message-Id: <20190530174819.1160221-10-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 09/12] eal/pci: Simplify rte_pci_get_iommu class by using a switch X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" Take several independent if statements and convert to a switch statement. Signed-off-by: Ben Walker Change-Id: Ia77c88ea484b529e8b0c9e09e8ef22cf3210e669 --- drivers/bus/pci/linux/pci.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index 11e2e4d1b..41fd82988 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -601,12 +601,12 @@ rte_pci_get_iommu_class(void) if (pci_ignore_device(dev)) continue; - if (dev->kdrv != RTE_KDRV_UNKNOWN && - dev->kdrv != RTE_KDRV_NONE) { + switch (dev->kdrv) { + case RTE_KDRV_UNKNOWN: + case RTE_KDRV_NONE: + break; + case RTE_KDRV_VFIO: is_bound = true; - } - - if (dev->kdrv == RTE_KDRV_VFIO) { FOREACH_DRIVER_ON_PCIBUS(drv) { if (!rte_pci_match(drv, dev)) continue; @@ -622,11 +622,14 @@ rte_pci_get_iommu_class(void) break; } } - } - - if (dev->kdrv == RTE_KDRV_IGB_UIO || - dev->kdrv == RTE_KDRV_UIO_GENERIC) { + break; + case RTE_KDRV_IGB_UIO: + case RTE_KDRV_UIO_GENERIC: + case RTE_KDRV_NIC_UIO: + is_bound = true; is_bound_uio = true; + break; + } } From patchwork Thu May 30 17:48:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53926 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id C94D21B9AE; Thu, 30 May 2019 19:48:53 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by dpdk.org (Postfix) with ESMTP id B10684CA6 for ; Thu, 30 May 2019 19:48:30 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga105.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020112" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:27 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:17 -0700 Message-Id: <20190530174819.1160221-11-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 10/12] eal/pci: Finding a device bound to UIO does not force PA X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" If a device is found that is bound to the UIO driver, only force IOVA_PA if there is a driver registered to use it. Signed-off-by: Ben Walker Change-Id: I8015f11a33ab1b7662bf374d6944eff8d7a74a07 --- drivers/bus/pci/linux/pci.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index 41fd82988..09af66571 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -627,7 +627,13 @@ rte_pci_get_iommu_class(void) case RTE_KDRV_UIO_GENERIC: case RTE_KDRV_NIC_UIO: is_bound = true; - is_bound_uio = true; + FOREACH_DRIVER_ON_PCIBUS(drv) { + if (!rte_pci_match(drv, dev)) + continue; + + is_bound_uio = true; + break; + } break; } From patchwork Thu May 30 17:48:18 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53917 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id B3A491B94F; Thu, 30 May 2019 19:48:37 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id 18A74378B for ; Thu, 30 May 2019 19:48:28 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020115" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:28 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:18 -0700 Message-Id: <20190530174819.1160221-12-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 11/12] eal/pci: rte_pci_get_iommu_class handles no drivers X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" In the case where no drivers are registered with the system, rte_pci_get_iommu_class should return RTE_IOVA_DC. Signed-off-by: Ben Walker Change-Id: Ia5b0cae100cfcfe46a9e4996328f9746ce33cfd3 --- drivers/bus/pci/linux/pci.c | 79 ++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c index 09af66571..abc21061f 100644 --- a/drivers/bus/pci/linux/pci.c +++ b/drivers/bus/pci/linux/pci.c @@ -589,49 +589,68 @@ pci_ignore_device(struct rte_pci_device *dev) enum rte_iova_mode rte_pci_get_iommu_class(void) { - bool is_bound = false; - bool is_vfio_noiommu_enabled = true; - bool has_iova_va = false; - bool is_bound_uio = false; - bool iommu_no_va = false; - struct rte_pci_device *dev = NULL; - struct rte_pci_driver *drv = NULL; + struct rte_pci_device *dev; + struct rte_pci_driver *drv; + struct rte_pci_addr *addr; + enum rte_iova_mode iova_mode; + + iova_mode = RTE_IOVA_DC; FOREACH_DEVICE_ON_PCIBUS(dev) { if (pci_ignore_device(dev)) continue; + addr = &dev->addr; + switch (dev->kdrv) { case RTE_KDRV_UNKNOWN: case RTE_KDRV_NONE: break; case RTE_KDRV_VFIO: - is_bound = true; FOREACH_DRIVER_ON_PCIBUS(drv) { if (!rte_pci_match(drv, dev)) continue; - /* - * just one PCI device needs to be checked out because - * the IOMMU hardware is the same for all of them. - */ - iommu_no_va = !pci_one_device_iommu_support_va(dev); + if ((drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) == 0) + continue; - if (drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { - has_iova_va = true; - break; + if (!pci_one_device_iommu_support_va(dev)) { + RTE_LOG(WARNING, EAL, "Device " PCI_PRI_FMT " wanted IOVA as VA, but ", + addr->domain, addr->bus, addr->devid, addr->function); + RTE_LOG(WARNING, EAL, "IOMMU does not support it.\n"); + iova_mode = RTE_IOVA_PA; + } +#ifdef VFIO_PRESENT + else if (rte_vfio_noiommu_is_enabled()) { + RTE_LOG(WARNING, EAL, "Device " PCI_PRI_FMT " wanted IOVA as VA, but ", + addr->domain, addr->bus, addr->devid, addr->function); + RTE_LOG(WARNING, EAL, "vfio-noiommu is enabled.\n"); + iova_mode = RTE_IOVA_PA; +#endif + } else if (iova_mode == RTE_IOVA_PA) { + RTE_LOG(WARNING, EAL, "Device " PCI_PRI_FMT " wanted IOVA as VA, but ", + addr->domain, addr->bus, addr->devid, addr->function); + RTE_LOG(WARNING, EAL, "other devices require PA.\n"); + } else { + iova_mode = RTE_IOVA_VA; } } break; case RTE_KDRV_IGB_UIO: case RTE_KDRV_UIO_GENERIC: case RTE_KDRV_NIC_UIO: - is_bound = true; FOREACH_DRIVER_ON_PCIBUS(drv) { if (!rte_pci_match(drv, dev)) continue; - is_bound_uio = true; + if (iova_mode == RTE_IOVA_VA) { + RTE_LOG(WARNING, EAL, "Some devices wanted IOVA as VA, but "); + RTE_LOG(WARNING, EAL, "device " PCI_PRI_FMT " requires PA.\n", + addr->domain, addr->bus, addr->devid, addr->function); + + } + + iova_mode = RTE_IOVA_PA; break; } break; @@ -639,29 +658,7 @@ rte_pci_get_iommu_class(void) } } - if (!is_bound) - return RTE_IOVA_DC; - -#ifdef VFIO_PRESENT - is_vfio_noiommu_enabled = rte_vfio_noiommu_is_enabled() == true ? - true : false; -#endif - - if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled && - !iommu_no_va) - return RTE_IOVA_VA; - - if (has_iova_va) { - RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa will be used because.. "); - if (is_vfio_noiommu_enabled) - RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n"); - if (is_bound_uio) - RTE_LOG(WARNING, EAL, "few device bound to UIO\n"); - if (iommu_no_va) - RTE_LOG(WARNING, EAL, "IOMMU does not support IOVA as VA\n"); - } - - return RTE_IOVA_PA; + return iova_mode; } /* Read PCI config space. */ From patchwork Thu May 30 17:48:19 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Walker, Benjamin" X-Patchwork-Id: 53924 Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id A1B4D1B9A4; Thu, 30 May 2019 19:48:50 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by dpdk.org (Postfix) with ESMTP id E095E1B945 for ; Thu, 30 May 2019 19:48:29 +0200 (CEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga002.fm.intel.com ([10.253.24.26]) by fmsmga101.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 30 May 2019 10:48:28 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,532,1549958400"; d="scan'208";a="180020118" Received: from bwalker-desk.ch.intel.com ([143.182.136.145]) by fmsmga002.fm.intel.com with ESMTP; 30 May 2019 10:48:28 -0700 From: Ben Walker To: dev@dpdk.org Cc: Ben Walker Date: Thu, 30 May 2019 10:48:19 -0700 Message-Id: <20190530174819.1160221-13-benjamin.walker@intel.com> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190530174819.1160221-1-benjamin.walker@intel.com> References: <20190530174819.1160221-1-benjamin.walker@intel.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 12/12] eal: If bus can't decide PA or VA, try to access PA X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" If the bus can't determine a preference for IOVA_PA vs. IOVA_VA by looking at the devices and drivers, as a last resort test if physical addresses are even accessible in /proc/self/pagemap. If they are, use IOVA_PA. If they are not, use IOVA_VA. Change-Id: If1eeb723283b80b24bd973987054fdad62f59cbd --- lib/librte_eal/common/eal_common_bus.c | 4 ---- lib/librte_eal/linux/eal/eal.c | 28 +++++++++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/librte_eal/common/eal_common_bus.c b/lib/librte_eal/common/eal_common_bus.c index c8f1901f0..77f1be1b4 100644 --- a/lib/librte_eal/common/eal_common_bus.c +++ b/lib/librte_eal/common/eal_common_bus.c @@ -237,10 +237,6 @@ rte_bus_get_iommu_class(void) mode |= bus->get_iommu_class(); } - if (mode != RTE_IOVA_VA) { - /* Use default IOVA mode */ - mode = RTE_IOVA_PA; - } return mode; } diff --git a/lib/librte_eal/linux/eal/eal.c b/lib/librte_eal/linux/eal/eal.c index 161399619..283aae120 100644 --- a/lib/librte_eal/linux/eal/eal.c +++ b/lib/librte_eal/linux/eal/eal.c @@ -948,6 +948,7 @@ rte_eal_init(int argc, char **argv) static char logid[PATH_MAX]; char cpuset[RTE_CPU_AFFINITY_STR_LEN]; char thread_name[RTE_MAX_THREAD_NAME_LEN]; + enum rte_iova_mode iova_mode; /* checks if the machine is adequate */ if (!rte_cpu_is_supported()) { @@ -1037,18 +1038,31 @@ rte_eal_init(int argc, char **argv) /* if no EAL option "--iova-mode=", use bus IOVA scheme */ if (internal_config.iova_mode == RTE_IOVA_DC) { - /* autodetect the IOVA mapping mode (default is RTE_IOVA_PA) */ - rte_eal_get_configuration()->iova_mode = - rte_bus_get_iommu_class(); + /* autodetect the IOVA mapping mode */ + iova_mode = rte_bus_get_iommu_class(); /* Workaround for KNI which requires physical address to work */ - if (rte_eal_get_configuration()->iova_mode == RTE_IOVA_VA && + if (iova_mode == RTE_IOVA_VA && rte_eal_check_module("rte_kni") == 1) { - rte_eal_get_configuration()->iova_mode = RTE_IOVA_PA; + iova_mode = RTE_IOVA_PA; RTE_LOG(WARNING, EAL, - "Some devices want IOVA as VA but PA will be used because.. " - "KNI module inserted\n"); + "Some devices want IOVA as VA but PA will be" + " used because KNI module inserted\n"); + } + + if (iova_mode == RTE_IOVA_DC) { + /* If the bus doesn't care, check if physical addresses are + * accessible. */ + if (rte_eal_using_phys_addrs()) { + /* Physical addresses are available, so the safest + * choice is to use those. */ + iova_mode = RTE_IOVA_PA; + } else { + iova_mode = RTE_IOVA_VA; + } } + + rte_eal_get_configuration()->iova_mode = iova_mode; } else { rte_eal_get_configuration()->iova_mode = internal_config.iova_mode;