@@ -109,7 +109,7 @@ TAILQ_HEAD(uio_res_list, uio_resource);
static struct uio_res_list *uio_res_list = NULL;
/* unbind kernel driver for this device */
-static int
+int
pci_unbind_kernel_driver(struct rte_pci_device *dev __rte_unused)
{
RTE_LOG(ERR, EAL, "RTE_PCI_DRV_FORCE_UNBIND flag is not implemented "
@@ -117,45 +117,6 @@ pci_unbind_kernel_driver(struct rte_pci_device *dev __rte_unused)
return -ENOTSUP;
}
-/* map a particular resource from a file */
-static void *
-pci_map_resource(void *requested_addr, const char *devname, off_t offset,
- size_t size)
-{
- int fd;
- void *mapaddr;
-
- /*
- * open devname, to mmap it
- */
- fd = open(devname, O_RDWR);
- if (fd < 0) {
- RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
- devname, strerror(errno));
- goto fail;
- }
-
- /* Map the PCI memory resource of device */
- mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE,
- MAP_SHARED, fd, offset);
- close(fd);
- if (mapaddr == MAP_FAILED ||
- (requested_addr != NULL && mapaddr != requested_addr)) {
- RTE_LOG(ERR, EAL, "%s(): cannot mmap(%s(%d), %p, 0x%lx, 0x%lx):"
- " %s (%p)\n", __func__, devname, fd, requested_addr,
- (unsigned long)size, (unsigned long)offset,
- strerror(errno), mapaddr);
- goto fail;
- }
-
- RTE_LOG(DEBUG, EAL, " PCI memory mapped at %p\n", mapaddr);
-
- return mapaddr;
-
-fail:
- return NULL;
-}
-
static int
pci_uio_map_secondary(struct rte_pci_device *dev)
{
@@ -169,7 +130,7 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
continue;
for (i = 0; i != uio_res->nb_maps; i++) {
- if (pci_map_resource(uio_res->maps[i].addr,
+ if (pci_map_resource(uio_res->maps[i].addr, INT_MIN,
uio_res->path,
(off_t)uio_res->maps[i].offset,
(size_t)uio_res->maps[i].size)
@@ -187,7 +148,7 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
}
/* map the PCI resource of a PCI device in virtual memory */
-static int
+int
pci_uio_map_resource(struct rte_pci_device *dev)
{
int i, j;
@@ -253,7 +214,8 @@ pci_uio_map_resource(struct rte_pci_device *dev)
maps[j].phaddr = dev->mem_resource[i].phys_addr;
maps[j].size = dev->mem_resource[i].len;
if (maps[j].addr != NULL ||
- (mapaddr = pci_map_resource(NULL, devname, (off_t)offset,
+ (mapaddr = pci_map_resource(NULL, INT_MIN, devname,
+ (off_t)offset,
(size_t)maps[j].size)
) == NULL) {
rte_free(uio_res);
@@ -270,20 +232,6 @@ pci_uio_map_resource(struct rte_pci_device *dev)
return (0);
}
-/* Compare two PCI device addresses. */
-static int
-pci_addr_comparison(struct rte_pci_addr *addr, struct rte_pci_addr *addr2)
-{
- uint64_t dev_addr = (addr->domain << 24) + (addr->bus << 16) + (addr->devid << 8) + addr->function;
- uint64_t dev_addr2 = (addr2->domain << 24) + (addr2->bus << 16) + (addr2->devid << 8) + addr2->function;
-
- if (dev_addr > dev_addr2)
- return 1;
- else
- return 0;
-}
-
-
/* Scan one pci sysfs entry, and fill the devices list from it. */
static int
pci_scan_one(int dev_pci_fd, struct pci_conf *conf)
@@ -425,71 +373,6 @@ error:
return -1;
}
-/*
- * If vendor/device ID match, call the devinit() function of the
- * driver.
- */
-int
-rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr, struct rte_pci_device *dev)
-{
- struct rte_pci_id *id_table;
- int ret;
-
- for (id_table = dr->id_table ; id_table->vendor_id != 0; id_table++) {
-
- /* check if device's identifiers match the driver's ones */
- if (id_table->vendor_id != dev->id.vendor_id &&
- id_table->vendor_id != PCI_ANY_ID)
- continue;
- if (id_table->device_id != dev->id.device_id &&
- id_table->device_id != PCI_ANY_ID)
- continue;
- if (id_table->subsystem_vendor_id != dev->id.subsystem_vendor_id &&
- id_table->subsystem_vendor_id != PCI_ANY_ID)
- continue;
- if (id_table->subsystem_device_id != dev->id.subsystem_device_id &&
- id_table->subsystem_device_id != PCI_ANY_ID)
- continue;
-
- struct rte_pci_addr *loc = &dev->addr;
-
- RTE_LOG(DEBUG, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n",
- loc->domain, loc->bus, loc->devid, loc->function,
- dev->numa_node);
-
- RTE_LOG(DEBUG, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id,
- dev->id.device_id, dr->name);
-
- /* no initialization when blacklisted, return without error */
- if (dev->devargs != NULL &&
- dev->devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) {
-
- RTE_LOG(DEBUG, EAL, " Device is blacklisted, not initializing\n");
- return 0;
- }
-
- if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
- /* map resources for devices that use igb_uio */
- ret = pci_uio_map_resource(dev);
- if (ret != 0)
- return ret;
- } else if (dr->drv_flags & RTE_PCI_DRV_FORCE_UNBIND &&
- rte_eal_process_type() == RTE_PROC_PRIMARY) {
- /* unbind current driver */
- if (pci_unbind_kernel_driver(dev) < 0)
- return -1;
- }
-
- /* reference driver structure */
- dev->driver = dr;
-
- /* call the driver devinit() function */
- return dr->devinit(dr, dev);
- }
- /* return positive value if driver is not found */
- return 1;
-}
-
/* Init the PCI EAL subsystem */
int
rte_eal_pci_init(void)
@@ -61,11 +61,15 @@
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
-#include <string.h>
-#include <inttypes.h>
-#include <stdint.h>
-#include <stdlib.h>
#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <sys/file.h>
+#include <stddef.h>
+#include <sys/mman.h>
#include <sys/queue.h>
#include <rte_interrupts.h>
@@ -154,8 +158,9 @@ rte_eal_pci_probe(void)
ret = pci_probe_all_drivers(dev);
if (ret < 0)
rte_exit(EXIT_FAILURE, "Requested device " PCI_PRI_FMT
- " cannot be used\n", dev->addr.domain, dev->addr.bus,
- dev->addr.devid, dev->addr.function);
+ " cannot be used\n",
+ dev->addr.domain, dev->addr.bus,
+ dev->addr.devid, dev->addr.function);
}
return 0;
@@ -205,3 +210,157 @@ rte_eal_pci_unregister(struct rte_pci_driver *driver)
{
TAILQ_REMOVE(&pci_driver_list, driver, next);
}
+
+/* map a particular resource from a file */
+void *
+pci_map_resource(void *requested_addr, const int vfio_fd,
+ const char *devname, off_t offset, size_t size)
+{
+ int fd;
+ void *mapaddr;
+
+ /*
+ * For non VFIO environment, vfio_fd is INT_MIN.
+ */
+ if (vfio_fd == INT_MIN) {
+ /*
+ * open devname, to mmap it
+ */
+ fd = open(devname, O_RDWR);
+ if (fd < 0) {
+ RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
+ devname, strerror(errno));
+ goto fail;
+ }
+ } else {
+ fd = vfio_fd;
+ }
+
+ /* Map the PCI memory resource of device */
+ mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, fd, offset);
+ /*
+ * Close applies only for non VFIO.
+ */
+ if (vfio_fd == INT_MIN)
+ close(fd);
+
+ if (mapaddr == MAP_FAILED ||
+ (requested_addr != NULL && mapaddr != requested_addr)) {
+ RTE_LOG(ERR, EAL,
+ "%s(): cannot mmap(%s(%d), %p, 0x%lx, 0x%lx): %s (%p)\n",
+ __func__, (devname ? devname : "vfio"), fd, requested_addr,
+ (unsigned long)size, (unsigned long)offset,
+ strerror(errno), mapaddr);
+ goto fail;
+ }
+
+ RTE_LOG(DEBUG, EAL, " PCI memory mapped at %p\n", mapaddr);
+
+ return mapaddr;
+
+fail:
+ return NULL;
+}
+
+/* Compare two PCI device addresses. */
+int
+pci_addr_comparison(struct rte_pci_addr *addr,
+ struct rte_pci_addr *addr2)
+{
+ uint64_t dev_addr = (addr->domain << 24) + (addr->bus << 16) +
+ (addr->devid << 8) + addr->function;
+ uint64_t dev_addr2 = (addr2->domain << 24) + (addr2->bus << 16) +
+ (addr2->devid << 8) + addr2->function;
+
+ if (dev_addr > dev_addr2)
+ return 1;
+ else
+ return 0;
+}
+
+/*
+ * If vendor/device ID match, call the devinit() function of the
+ * driver.
+ */
+int
+rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr,
+ struct rte_pci_device *dev)
+{
+ struct rte_pci_id *id_table;
+ int ret;
+
+ for (id_table = dr->id_table ; id_table->vendor_id != 0; id_table++) {
+
+ /* check if device's identifiers match the driver's ones */
+ if (id_table->vendor_id != dev->id.vendor_id &&
+ id_table->vendor_id != PCI_ANY_ID)
+ continue;
+ if (id_table->device_id != dev->id.device_id &&
+ id_table->device_id != PCI_ANY_ID)
+ continue;
+ if (id_table->subsystem_vendor_id !=
+ dev->id.subsystem_vendor_id &&
+ id_table->subsystem_vendor_id != PCI_ANY_ID)
+ continue;
+ if (id_table->subsystem_device_id !=
+ dev->id.subsystem_device_id &&
+ id_table->subsystem_device_id != PCI_ANY_ID)
+ continue;
+
+ struct rte_pci_addr *loc = &dev->addr;
+
+ RTE_LOG(DEBUG, EAL,
+ "PCI device "PCI_PRI_FMT" on NUMA socket %i\n",
+ loc->domain, loc->bus, loc->devid, loc->function,
+ dev->numa_node);
+
+ RTE_LOG(DEBUG, EAL,
+ " probe driver: %x:%x %s\n", dev->id.vendor_id,
+ dev->id.device_id, dr->name);
+
+ /* no initialization when blacklisted, return without error */
+ if (dev->devargs != NULL &&
+ dev->devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) {
+
+ RTE_LOG(DEBUG, EAL,
+ " Device is blacklisted, not initializing\n");
+ return 0;
+ }
+
+ if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
+
+#ifdef RTE_EXEC_ENV_BSDAPP
+ /* map resources for devices that use igb_uio */
+ ret = pci_uio_map_resource(dev);
+#else /* RTE_EXEC_ENV_BSDAPP */
+#ifdef RTE_PCI_CONFIG
+ /*
+ * Set PCIe config space for high performance.
+ * Return value can be ignored.
+ */
+ pci_config_space_set(dev);
+#endif /* RTE_PCI_CONFIG */
+ /* map resources for devices that use igb_uio */
+ ret = pci_map_device(dev);
+
+#endif /* RTE_EXEC_ENV_BSDAPP */
+
+ if (ret != 0)
+ return ret;
+ } else if (dr->drv_flags & RTE_PCI_DRV_FORCE_UNBIND &&
+ rte_eal_process_type() == RTE_PROC_PRIMARY) {
+ /* unbind current driver */
+ if (pci_unbind_kernel_driver(dev) < 0)
+ return -1;
+ }
+
+ /* reference driver structure */
+ dev->driver = dr;
+
+ /* call the driver devinit() function */
+ return dr->devinit(dr, dev);
+ }
+ /* return positive value if driver is not found */
+ return 1;
+}
@@ -37,6 +37,7 @@
#include <stdio.h>
#include <rte_eal.h>
+#include <rte_pci.h>
/**
* Initialize the memzone subsystem (private to eal).
@@ -348,4 +349,63 @@ int rte_eal_hugepage_init(void);
*/
int rte_eal_hugepage_attach(void);
+/**
+ * This function maps a particular resource from a file
+ *
+ * This function is private to the EAL.
+ */
+void *
+pci_map_resource(void *requested_addr, const int vfio_fd,
+ const char *devname, off_t offset, size_t size);
+
+/**
+ * This function compares two PCI device addresses
+ *
+ * This function is private to the EAL.
+ */
+int
+pci_addr_comparison(struct rte_pci_addr *addr,
+ struct rte_pci_addr *addr2);
+
+/**
+ * This function maps PCI resource of a PCI
+ * device in virtual memory
+ *
+ * This function is private to the EAL.
+ */
+int
+pci_uio_map_resource(struct rte_pci_device *dev);
+
+/**
+ * This function unbinds kernel driver for this device
+ *
+ * This function is private to the EAL.
+ */
+int
+pci_unbind_kernel_driver(struct rte_pci_device *dev);
+
+#ifdef RTE_EXEC_ENV_LINUXAPP
+#ifdef RTE_PCI_CONFIG
+/**
+ * This function sets PCIe config space for
+ * high performance
+ *
+ * This function is private to the EAL.
+ */
+void
+pci_config_space_set(struct rte_pci_device *dev);
+
+#endif /* RTE_PCI_CONFIG */
+
+/**
+ * This function maps resources for devices
+ * that use igb_uio
+ *
+ * This function is private to the EAL.
+ */
+int
+pci_map_device(struct rte_pci_device *dev);
+
+#endif /* RTE_EXEC_ENV_LINUXAPP */
+
#endif /* _EAL_PRIVATE_H_ */
@@ -59,7 +59,7 @@
struct mapped_pci_res_list *pci_res_list = NULL;
/* unbind kernel driver for this device */
-static int
+int
pci_unbind_kernel_driver(struct rte_pci_device *dev)
{
int n;
@@ -115,28 +115,6 @@ pci_find_max_end_va(void)
return RTE_PTR_ADD(last->addr, last->len);
}
-
-/* map a particular resource from a file */
-void *
-pci_map_resource(void *requested_addr, int fd, off_t offset, size_t size)
-{
- void *mapaddr;
-
- /* Map the PCI memory resource of device */
- mapaddr = mmap(requested_addr, size, PROT_READ | PROT_WRITE,
- MAP_SHARED, fd, offset);
- if (mapaddr == MAP_FAILED) {
- RTE_LOG(ERR, EAL, "%s(): cannot mmap(%d, %p, 0x%lx, 0x%lx): %s (%p)\n",
- __func__, fd, requested_addr,
- (unsigned long)size, (unsigned long)offset,
- strerror(errno), mapaddr);
- } else {
- RTE_LOG(DEBUG, EAL, " PCI memory mapped at %p\n", mapaddr);
- }
-
- return mapaddr;
-}
-
/* parse the "resource" sysfs file */
#define IORESOURCE_MEM 0x00000200
@@ -200,20 +178,6 @@ error:
return -1;
}
-/* Compare two PCI device addresses. */
-static int
-pci_addr_comparison(struct rte_pci_addr *addr, struct rte_pci_addr *addr2)
-{
- uint64_t dev_addr = (addr->domain << 24) + (addr->bus << 16) + (addr->devid << 8) + addr->function;
- uint64_t dev_addr2 = (addr2->domain << 24) + (addr2->bus << 16) + (addr2->devid << 8) + addr2->function;
-
- if (dev_addr > dev_addr2)
- return 1;
- else
- return 0;
-}
-
-
/* Scan one pci sysfs entry, and fill the devices list from it. */
static int
pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus,
@@ -483,7 +447,7 @@ pci_config_max_read_request_size(struct rte_pci_device *dev)
return 0;
}
-static void
+void
pci_config_space_set(struct rte_pci_device *dev)
{
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
@@ -497,7 +461,7 @@ pci_config_space_set(struct rte_pci_device *dev)
}
#endif
-static int
+int
pci_map_device(struct rte_pci_device *dev)
{
int ret, mapped = 0;
@@ -521,77 +485,6 @@ pci_map_device(struct rte_pci_device *dev)
return 0;
}
-/*
- * If vendor/device ID match, call the devinit() function of the
- * driver.
- */
-int
-rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr, struct rte_pci_device *dev)
-{
- int ret;
- struct rte_pci_id *id_table;
-
- for (id_table = dr->id_table ; id_table->vendor_id != 0; id_table++) {
-
- /* check if device's identifiers match the driver's ones */
- if (id_table->vendor_id != dev->id.vendor_id &&
- id_table->vendor_id != PCI_ANY_ID)
- continue;
- if (id_table->device_id != dev->id.device_id &&
- id_table->device_id != PCI_ANY_ID)
- continue;
- if (id_table->subsystem_vendor_id != dev->id.subsystem_vendor_id &&
- id_table->subsystem_vendor_id != PCI_ANY_ID)
- continue;
- if (id_table->subsystem_device_id != dev->id.subsystem_device_id &&
- id_table->subsystem_device_id != PCI_ANY_ID)
- continue;
-
- struct rte_pci_addr *loc = &dev->addr;
-
- RTE_LOG(DEBUG, EAL, "PCI device "PCI_PRI_FMT" on NUMA socket %i\n",
- loc->domain, loc->bus, loc->devid, loc->function,
- dev->numa_node);
-
- RTE_LOG(DEBUG, EAL, " probe driver: %x:%x %s\n", dev->id.vendor_id,
- dev->id.device_id, dr->name);
-
- /* no initialization when blacklisted, return without error */
- if (dev->devargs != NULL &&
- dev->devargs->type == RTE_DEVTYPE_BLACKLISTED_PCI) {
- RTE_LOG(DEBUG, EAL, " Device is blacklisted, not initializing\n");
- return 1;
- }
-
- if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) {
-#ifdef RTE_PCI_CONFIG
- /*
- * Set PCIe config space for high performance.
- * Return value can be ignored.
- */
- pci_config_space_set(dev);
-#endif
- /* map resources for devices that use igb_uio */
- ret = pci_map_device(dev);
- if (ret != 0)
- return ret;
- } else if (dr->drv_flags & RTE_PCI_DRV_FORCE_UNBIND &&
- rte_eal_process_type() == RTE_PROC_PRIMARY) {
- /* unbind current driver */
- if (pci_unbind_kernel_driver(dev) < 0)
- return -1;
- }
-
- /* reference driver structure */
- dev->driver = dr;
-
- /* call the driver devinit() function */
- return dr->devinit(dr, dev);
- }
- /* return positive value if driver is not found */
- return 1;
-}
-
/* Init the PCI EAL subsystem */
int
rte_eal_pci_init(void)
@@ -65,12 +65,6 @@ extern struct mapped_pci_res_list *pci_res_list;
extern void *pci_map_addr;
void *pci_find_max_end_va(void);
-void *pci_map_resource(void *requested_addr, int fd, off_t offset,
- size_t size);
-
-/* map IGB_UIO resource prototype */
-int pci_uio_map_resource(struct rte_pci_device *dev);
-
#ifdef VFIO_PRESENT
#define VFIO_MAX_GROUPS 64
@@ -46,6 +46,7 @@
#include "rte_pci_dev_ids.h"
#include "eal_filesystem.h"
#include "eal_pci_init.h"
+#include "eal_private.h"
static int pci_parse_sysfs_value(const char *filename, uint64_t *val);
@@ -117,7 +118,7 @@ pci_uio_get_mappings(const char *devname, struct pci_map maps[], int nb_maps)
static int
pci_uio_map_secondary(struct rte_pci_device *dev)
{
- int fd, i;
+ int i;
struct mapped_pci_resource *uio_res;
TAILQ_FOREACH(uio_res, pci_res_list, next) {
@@ -127,27 +128,15 @@ pci_uio_map_secondary(struct rte_pci_device *dev)
continue;
for (i = 0; i != uio_res->nb_maps; i++) {
- /*
- * open devname, to mmap it
- */
- fd = open(uio_res->path, O_RDWR);
- if (fd < 0) {
- RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
- uio_res->path, strerror(errno));
- return -1;
- }
-
- if (pci_map_resource(uio_res->maps[i].addr, fd,
+ if (pci_map_resource(uio_res->maps[i].addr, INT_MIN,
+ uio_res->path,
(off_t)uio_res->maps[i].offset,
(size_t)uio_res->maps[i].size)
!= uio_res->maps[i].addr) {
RTE_LOG(ERR, EAL,
"Cannot mmap device resource\n");
- close(fd);
return -1;
}
- /* fd is not needed in slave process, close it */
- close(fd);
}
return 0;
}
@@ -344,7 +333,6 @@ pci_uio_map_resource(struct rte_pci_device *dev)
maps = uio_res->maps;
for (i = 0; i != PCI_MAX_RESOURCE; i++) {
- int fd;
/* skip empty BAR */
phaddr = dev->mem_resource[i].phys_addr;
@@ -361,16 +349,6 @@ pci_uio_map_resource(struct rte_pci_device *dev)
int fail = 0;
offset = j * pagesz;
- /*
- * open devname, to mmap it
- */
- fd = open(devname, O_RDWR);
- if (fd < 0) {
- RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
- devname, strerror(errno));
- return -1;
- }
-
if (maps[j].addr != NULL)
fail = 1;
else {
@@ -378,7 +356,9 @@ pci_uio_map_resource(struct rte_pci_device *dev)
if (pci_map_addr == NULL)
pci_map_addr = pci_find_max_end_va();
- mapaddr = pci_map_resource(pci_map_addr, fd, (off_t)offset,
+ mapaddr = pci_map_resource(pci_map_addr,
+ INT_MIN, devname,
+ (off_t)offset,
(size_t)maps[j].size);
if (mapaddr == MAP_FAILED)
fail = 1;
@@ -388,10 +368,8 @@ pci_uio_map_resource(struct rte_pci_device *dev)
if (fail) {
rte_free(uio_res);
- close(fd);
return -1;
}
- close(fd);
maps[j].addr = mapaddr;
maps[j].offset = offset;
@@ -48,6 +48,7 @@
#include "eal_filesystem.h"
#include "eal_pci_init.h"
#include "eal_vfio.h"
+#include "eal_private.h"
/**
* @file
@@ -726,12 +727,12 @@ pci_vfio_map_resource(struct rte_pci_device *dev)
if (pci_map_addr == NULL)
pci_map_addr = pci_find_max_end_va();
- bar_addr = pci_map_resource(pci_map_addr, vfio_dev_fd, reg.offset,
- reg.size);
+ bar_addr = pci_map_resource(pci_map_addr, vfio_dev_fd,
+ NULL, reg.offset, reg.size);
pci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg.size);
} else {
- bar_addr = pci_map_resource(maps[i].addr, vfio_dev_fd, reg.offset,
- reg.size);
+ bar_addr = pci_map_resource(maps[i].addr, vfio_dev_fd,
+ NULL, reg.offset, reg.size);
}
if (bar_addr == MAP_FAILED ||