Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/43448/?format=api
{ "id": 43448, "url": "http://patchwork.dpdk.org/api/patches/43448/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/b38b240d6b11f3f7dc6551b3d2232b5074a1202f.1532949065.git.anatoly.burakov@intel.com/", "project": { "id": 1, "url": "http://patchwork.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<b38b240d6b11f3f7dc6551b3d2232b5074a1202f.1532949065.git.anatoly.burakov@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/b38b240d6b11f3f7dc6551b3d2232b5074a1202f.1532949065.git.anatoly.burakov@intel.com", "date": "2018-07-30T11:17:03", "name": "[18.11] pci/vfio: allow mapping MSI-X BARs if kernel allows it", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "88328c6c87b0fba8f298e9b25f6141d8191e4f61", "submitter": { "id": 4, "url": "http://patchwork.dpdk.org/api/people/4/?format=api", "name": "Anatoly Burakov", "email": "anatoly.burakov@intel.com" }, "delegate": null, "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/b38b240d6b11f3f7dc6551b3d2232b5074a1202f.1532949065.git.anatoly.burakov@intel.com/mbox/", "series": [ { "id": 819, "url": "http://patchwork.dpdk.org/api/series/819/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=819", "date": "2018-07-30T11:17:03", "name": "[18.11] pci/vfio: allow mapping MSI-X BARs if kernel allows it", "version": 1, "mbox": "http://patchwork.dpdk.org/series/819/mbox/" } ], "comments": "http://patchwork.dpdk.org/api/patches/43448/comments/", "check": "success", "checks": "http://patchwork.dpdk.org/api/patches/43448/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 06EA05699;\n\tMon, 30 Jul 2018 13:17:11 +0200 (CEST)", "from mga07.intel.com (mga07.intel.com [134.134.136.100])\n\tby dpdk.org (Postfix) with ESMTP id 470CB5699\n\tfor <dev@dpdk.org>; Mon, 30 Jul 2018 13:17:08 +0200 (CEST)", "from orsmga006.jf.intel.com ([10.7.209.51])\n\tby orsmga105.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t30 Jul 2018 04:17:05 -0700", "from irvmail001.ir.intel.com ([163.33.26.43])\n\tby orsmga006.jf.intel.com with ESMTP; 30 Jul 2018 04:17:04 -0700", "from sivswdev01.ir.intel.com (sivswdev01.ir.intel.com\n\t[10.237.217.45])\n\tby irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id\n\tw6UBH3Ca017397; Mon, 30 Jul 2018 12:17:03 +0100", "from sivswdev01.ir.intel.com (localhost [127.0.0.1])\n\tby sivswdev01.ir.intel.com with ESMTP id w6UBH3DD024949;\n\tMon, 30 Jul 2018 12:17:03 +0100", "(from aburakov@localhost)\n\tby sivswdev01.ir.intel.com with LOCAL id w6UBH3MI024943;\n\tMon, 30 Jul 2018 12:17:03 +0100" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.51,422,1526367600\"; d=\"scan'208\";a=\"61842260\"", "From": "Anatoly Burakov <anatoly.burakov@intel.com>", "To": "dev@dpdk.org", "Cc": "jerin.jacob@caviumnetworks.com, thomas@monjalon.net,\n\tt.yoshimura8869@gmail.com", "Date": "Mon, 30 Jul 2018 12:17:03 +0100", "Message-Id": "<b38b240d6b11f3f7dc6551b3d2232b5074a1202f.1532949065.git.anatoly.burakov@intel.com>", "X-Mailer": "git-send-email 1.7.0.7", "Subject": "[dpdk-dev] [PATCH 18.11] pci/vfio: allow mapping MSI-X BARs if\n\tkernel allows it", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Currently, DPDK will skip mapping some areas (or even an entire BAR)\nif MSI-X happens to be in it but is smaller than page address.\n\nKernels 4.16+ will allow mapping MSI-X BARs [1], and will report this\nas a capability flag. Capability flags themselves are also only\nsupported since kernel 4.6 [2].\n\nThis commit will introduce support for checking VFIO capabilities,\nand will use it to check if we are allowed to map BARs with MSI-X\ntables in them, along with backwards compatibility for older\nkernels, including a workaround for a variable rename in VFIO\nregion info structure [3].\n\n[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/\nlinux.git/commit/?id=a32295c612c57990d17fb0f41e7134394b2f35f6\n\n[2] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/\nlinux.git/commit/?id=c84982adb23bcf3b99b79ca33527cd2625fbe279\n\n[3] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/\nlinux.git/commit/?id=ff63eb638d63b95e489f976428f1df01391e15e4\n\nSigned-off-by: Anatoly Burakov <anatoly.burakov@intel.com>\n---\n drivers/bus/pci/linux/pci_vfio.c | 127 ++++++++++++++++++++---\n lib/librte_eal/common/include/rte_vfio.h | 26 +++++\n 2 files changed, 140 insertions(+), 13 deletions(-)", "diff": "diff --git a/drivers/bus/pci/linux/pci_vfio.c b/drivers/bus/pci/linux/pci_vfio.c\nindex 686386d6a..e7765ee11 100644\n--- a/drivers/bus/pci/linux/pci_vfio.c\n+++ b/drivers/bus/pci/linux/pci_vfio.c\n@@ -415,6 +415,88 @@ pci_vfio_mmap_bar(int vfio_dev_fd, struct mapped_pci_resource *vfio_res,\n \treturn 0;\n }\n \n+/*\n+ * region info may contain capability headers, so we need to keep reallocating\n+ * the memory until we match allocated memory size with argsz.\n+ */\n+static int\n+pci_vfio_get_region_info(int vfio_dev_fd, struct vfio_region_info **info,\n+\t\tint region)\n+{\n+\tstruct vfio_region_info *ri;\n+\tsize_t argsz = sizeof(*ri);\n+\tint ret;\n+\n+\tri = malloc(sizeof(*ri));\n+\tif (ri == NULL) {\n+\t\tRTE_LOG(ERR, EAL, \"Cannot allocate memory for region info\\n\");\n+\t\treturn -1;\n+\t}\n+again:\n+\tmemset(ri, 0, argsz);\n+\tri->argsz = argsz;\n+\tri->index = region;\n+\n+\tret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, info);\n+\tif (ret) {\n+\t\tfree(ri);\n+\t\treturn ret;\n+\t}\n+\tif (ri->argsz != argsz) {\n+\t\targsz = ri->argsz;\n+\t\tri = realloc(ri, argsz);\n+\n+\t\tif (ri == NULL) {\n+\t\t\tRTE_LOG(ERR, EAL, \"Cannot reallocate memory for region info\\n\");\n+\t\t\treturn -1;\n+\t\t}\n+\t\tgoto again;\n+\t}\n+\t*info = ri;\n+\n+\treturn 0;\n+}\n+\n+static struct vfio_info_cap_header *\n+pci_vfio_info_cap(struct vfio_region_info *info, int cap)\n+{\n+\tstruct vfio_info_cap_header *h;\n+\tsize_t offset;\n+\n+\tif ((info->flags & RTE_VFIO_INFO_FLAG_CAPS) == 0) {\n+\t\t/* VFIO info does not advertise capabilities */\n+\t\treturn NULL;\n+\t}\n+\n+\toffset = VFIO_CAP_OFFSET(info);\n+\twhile (offset != 0) {\n+\t\th = RTE_PTR_ADD(info, offset);\n+\t\tif (h->id == cap)\n+\t\t\treturn h;\n+\t\toffset = h->next;\n+\t}\n+\treturn NULL;\n+}\n+\n+static int\n+pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)\n+{\n+\tstruct vfio_region_info *info;\n+\tint ret;\n+\n+\tret = pci_vfio_get_region_info(vfio_dev_fd, &info, msix_region);\n+\tif (ret < 0)\n+\t\treturn -1;\n+\n+\tret = pci_vfio_info_cap(info, RTE_VFIO_CAP_MSIX_MAPPABLE) != NULL;\n+\n+\t/* cleanup */\n+\tfree(info);\n+\n+\treturn ret;\n+}\n+\n+\n static int\n pci_vfio_map_resource_primary(struct rte_pci_device *dev)\n {\n@@ -464,56 +546,75 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)\n \tif (ret < 0) {\n \t\tRTE_LOG(ERR, EAL, \" %s cannot get MSI-X BAR number!\\n\",\n \t\t\t\tpci_addr);\n-\t\tgoto err_vfio_dev_fd;\n+\t\tgoto err_vfio_res;\n+\t}\n+\t/* if we found our MSI-X BAR region, check if we can mmap it */\n+\tif (vfio_res->msix_table.bar_index != -1) {\n+\t\tint ret = pci_vfio_msix_is_mappable(vfio_dev_fd,\n+\t\t\t\tvfio_res->msix_table.bar_index);\n+\t\tif (ret < 0) {\n+\t\t\tRTE_LOG(ERR, EAL, \"Couldn't check if MSI-X BAR is mappable\\n\");\n+\t\t\tgoto err_vfio_res;\n+\t\t} else if (ret != 0) {\n+\t\t\t/* we can map it, so we don't care where it is */\n+\t\t\tRTE_LOG(DEBUG, EAL, \"VFIO reports MSI-X BAR as mappable\\n\");\n+\t\t\tvfio_res->msix_table.bar_index = -1;\n+\t\t}\n \t}\n \n \tfor (i = 0; i < (int) vfio_res->nb_maps; i++) {\n-\t\tstruct vfio_region_info reg = { .argsz = sizeof(reg) };\n+\t\tstruct vfio_region_info *reg;\n \t\tvoid *bar_addr;\n \n-\t\treg.index = i;\n-\n-\t\tret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, ®);\n-\t\tif (ret) {\n+\t\tret = pci_vfio_get_region_info(vfio_dev_fd, ®, i);\n+\t\tif (ret < 0) {\n \t\t\tRTE_LOG(ERR, EAL, \" %s cannot get device region info \"\n-\t\t\t\t\t\"error %i (%s)\\n\", pci_addr, errno, strerror(errno));\n+\t\t\t\t\"error %i (%s)\\n\", pci_addr, errno,\n+\t\t\t\tstrerror(errno));\n \t\t\tgoto err_vfio_res;\n \t\t}\n \n \t\t/* chk for io port region */\n \t\tret = pci_vfio_is_ioport_bar(vfio_dev_fd, i);\n-\t\tif (ret < 0)\n+\t\tif (ret < 0) {\n+\t\t\tfree(reg);\n \t\t\tgoto err_vfio_res;\n-\t\telse if (ret) {\n+\t\t} else if (ret) {\n \t\t\tRTE_LOG(INFO, EAL, \"Ignore mapping IO port bar(%d)\\n\",\n \t\t\t\t\ti);\n+\t\t\tfree(reg);\n \t\t\tcontinue;\n \t\t}\n \n \t\t/* skip non-mmapable BARs */\n-\t\tif ((reg.flags & VFIO_REGION_INFO_FLAG_MMAP) == 0)\n+\t\tif ((reg->flags & VFIO_REGION_INFO_FLAG_MMAP) == 0) {\n+\t\t\tfree(reg);\n \t\t\tcontinue;\n+\t\t}\n \n \t\t/* try mapping somewhere close to the end of hugepages */\n \t\tif (pci_map_addr == NULL)\n \t\t\tpci_map_addr = pci_find_max_end_va();\n \n \t\tbar_addr = pci_map_addr;\n-\t\tpci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg.size);\n+\t\tpci_map_addr = RTE_PTR_ADD(bar_addr, (size_t) reg->size);\n \n \t\tmaps[i].addr = bar_addr;\n-\t\tmaps[i].offset = reg.offset;\n-\t\tmaps[i].size = reg.size;\n+\t\tmaps[i].offset = reg->offset;\n+\t\tmaps[i].size = reg->size;\n \t\tmaps[i].path = NULL; /* vfio doesn't have per-resource paths */\n \n \t\tret = pci_vfio_mmap_bar(vfio_dev_fd, vfio_res, i, 0);\n \t\tif (ret < 0) {\n \t\t\tRTE_LOG(ERR, EAL, \" %s mapping BAR%i failed: %s\\n\",\n \t\t\t\t\tpci_addr, i, strerror(errno));\n+\t\t\tfree(reg);\n \t\t\tgoto err_vfio_res;\n \t\t}\n \n \t\tdev->mem_resource[i].addr = maps[i].addr;\n+\n+\t\tfree(reg);\n \t}\n \n \tif (pci_rte_vfio_setup_device(dev, vfio_dev_fd) < 0) {\ndiff --git a/lib/librte_eal/common/include/rte_vfio.h b/lib/librte_eal/common/include/rte_vfio.h\nindex 5ca13fcce..f6617e004 100644\n--- a/lib/librte_eal/common/include/rte_vfio.h\n+++ b/lib/librte_eal/common/include/rte_vfio.h\n@@ -14,6 +14,8 @@\n extern \"C\" {\n #endif\n \n+#include <stdint.h>\n+\n /*\n * determine if VFIO is present on the system\n */\n@@ -44,6 +46,30 @@ extern \"C\" {\n #define RTE_VFIO_NOIOMMU 8\n #endif\n \n+/*\n+ * capabilities are only supported on kernel 4.6+. there were also some API\n+ * changes as well, so add a macro to get cap offset.\n+ */\n+#ifdef VFIO_REGION_INFO_FLAG_CAPS\n+#define RTE_VFIO_INFO_FLAG_CAPS VFIO_REGION_INFO_FLAG_CAPS\n+#define VFIO_CAP_OFFSET(x) (x->cap_offset)\n+#else\n+#define RTE_VFIO_INFO_FLAG_CAPS (1 << 3)\n+#define VFIO_CAP_OFFSET(x) (x->resv)\n+struct vfio_info_cap_header {\n+\tuint16_t id;\n+\tuint16_t version;\n+\tuint32_t next;\n+};\n+#endif\n+\n+/* kernels 4.16+ can map BAR containing MSI-X table */\n+#ifdef VFIO_REGION_INFO_CAP_MSIX_MAPPABLE\n+#define RTE_VFIO_CAP_MSIX_MAPPABLE VFIO_REGION_INFO_CAP_MSIX_MAPPABLE\n+#else\n+#define RTE_VFIO_CAP_MSIX_MAPPABLE 3\n+#endif\n+\n #else /* not VFIO_PRESENT */\n \n /* we don't need an actual definition, only pointer is used */\n", "prefixes": [ "18.11" ] }