[dpdk-dev] vfio: fix device unplug when several devices per vfio group
Checks
Commit Message
VFIO allows a secure way of assigning devices to user space and those
devices which can not be isolated from other ones are set in same VFIO
group. Releasing or unplugging a device should be aware of remaining
devices is the same group for avoiding to close such a group.
Fixes: 94c0776b1bad ("vfio: support hotplug")
Signed-off-by: Alejandro Lucero <alejandro.lucero@netronome.com>
---
lib/librte_eal/linuxapp/eal/eal_vfio.c | 32 ++++++++++++++++++++++++--------
lib/librte_eal/linuxapp/eal/eal_vfio.h | 1 +
2 files changed, 25 insertions(+), 8 deletions(-)
Comments
> From: Alejandro Lucero [mailto:alejandro.lucero@netronome.com]
> Sent: Wednesday, April 26, 2017 11:50 AM
> To: dev@dpdk.org
> Cc: Burakov, Anatoly <anatoly.burakov@intel.com>
> Subject: [PATCH] vfio: fix device unplug when several devices per vfio group
>
> VFIO allows a secure way of assigning devices to user space and those
> devices which can not be isolated from other ones are set in same VFIO
> group. Releasing or unplugging a device should be aware of remaining
> devices is the same group for avoiding to close such a group.
>
> Fixes: 94c0776b1bad ("vfio: support hotplug")
>
> Signed-off-by: Alejandro Lucero <alejandro.lucero@netronome.com>
> ---
> lib/librte_eal/linuxapp/eal/eal_vfio.c | 32 ++++++++++++++++++++++++----
> ---- lib/librte_eal/linuxapp/eal/eal_vfio.h | 1 +
> 2 files changed, 25 insertions(+), 8 deletions(-)
>
> diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c
> b/lib/librte_eal/linuxapp/eal/eal_vfio.c
> index 6e2e84c..6e24273 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_vfio.c
> +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c
> @@ -184,6 +184,7 @@
> if (vfio_cfg.vfio_groups[i].fd == vfio_group_fd) {
> vfio_cfg.vfio_groups[i].group_no = -1;
> vfio_cfg.vfio_groups[i].fd = -1;
> + vfio_cfg.vfio_groups[i].devices = 0;
> vfio_cfg.vfio_active_groups--;
> return 0;
> }
> @@ -353,6 +354,7 @@
> clear_group(vfio_group_fd);
> return -1;
> }
> + vfio_cfg.vfio_groups[vfio_group_fd].devices++;
>
> return 0;
> }
> @@ -390,17 +392,30 @@
> * code will unset the container and the IOMMU mappings.
> */
>
> - if (close(vfio_group_fd) < 0)
> - RTE_LOG(INFO, EAL, "Error when closing vfio_group_fd for
> %s\n",
> - dev_addr);
> -
> - if (close(vfio_dev_fd) < 0)
> + /* Closing a device */
> + if (close(vfio_dev_fd) < 0) {
> RTE_LOG(INFO, EAL, "Error when closing vfio_dev_fd for
> %s\n",
> dev_addr);
> + return -1;
> + }
>
> - if (clear_group(vfio_group_fd) < 0)
> - RTE_LOG(INFO, EAL, "Error when clearing group for %s\n",
> - dev_addr);
> + /* An VFIO group can have several devices attached. Just when there
> is
> + * no devices remaining should the group be closed.
> + */
> + if (--vfio_cfg.vfio_groups[vfio_group_fd].devices == 0) {
> +
> + if (close(vfio_group_fd) < 0) {
> + RTE_LOG(INFO, EAL, "Error when closing
> vfio_group_fd for %s\n",
> + dev_addr);
> + return -1;
> + }
> +
> + if (clear_group(vfio_group_fd) < 0) {
> + RTE_LOG(INFO, EAL, "Error when clearing group for
> %s\n",
> + dev_addr);
> + return -1;
> + }
> + }
>
> return 0;
> }
> @@ -415,6 +430,7 @@
> for (i = 0; i < VFIO_MAX_GROUPS; i++) {
> vfio_cfg.vfio_groups[i].fd = -1;
> vfio_cfg.vfio_groups[i].group_no = -1;
> + vfio_cfg.vfio_groups[i].devices = 0;
> }
>
> /* inform the user that we are probing for VFIO */ diff --git
> a/lib/librte_eal/linuxapp/eal/eal_vfio.h
> b/lib/librte_eal/linuxapp/eal/eal_vfio.h
> index 7fcec2c..2c7cb94 100644
> --- a/lib/librte_eal/linuxapp/eal/eal_vfio.h
> +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h
> @@ -103,6 +103,7 @@ struct vfio_iommu_spapr_tce_remove { struct
> vfio_group {
> int group_no;
> int fd;
> + int devices;
> };
>
> struct vfio_config {
> --
> 1.9.1
I have tested this on my setup on an old kernel with multiple attach/detaches, and it works (whereas it fails without this patch).
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
28/04/2017 15:25, Burakov, Anatoly:
> From: Alejandro Lucero [mailto:alejandro.lucero@netronome.com]
> > VFIO allows a secure way of assigning devices to user space and those
> > devices which can not be isolated from other ones are set in same VFIO
> > group. Releasing or unplugging a device should be aware of remaining
> > devices is the same group for avoiding to close such a group.
> >
> > Fixes: 94c0776b1bad ("vfio: support hotplug")
> >
> > Signed-off-by: Alejandro Lucero <alejandro.lucero@netronome.com>
>
> I have tested this on my setup on an old kernel with multiple attach/detaches, and it works (whereas it fails without this patch).
>
> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>
Applied, thanks
@@ -184,6 +184,7 @@
if (vfio_cfg.vfio_groups[i].fd == vfio_group_fd) {
vfio_cfg.vfio_groups[i].group_no = -1;
vfio_cfg.vfio_groups[i].fd = -1;
+ vfio_cfg.vfio_groups[i].devices = 0;
vfio_cfg.vfio_active_groups--;
return 0;
}
@@ -353,6 +354,7 @@
clear_group(vfio_group_fd);
return -1;
}
+ vfio_cfg.vfio_groups[vfio_group_fd].devices++;
return 0;
}
@@ -390,17 +392,30 @@
* code will unset the container and the IOMMU mappings.
*/
- if (close(vfio_group_fd) < 0)
- RTE_LOG(INFO, EAL, "Error when closing vfio_group_fd for %s\n",
- dev_addr);
-
- if (close(vfio_dev_fd) < 0)
+ /* Closing a device */
+ if (close(vfio_dev_fd) < 0) {
RTE_LOG(INFO, EAL, "Error when closing vfio_dev_fd for %s\n",
dev_addr);
+ return -1;
+ }
- if (clear_group(vfio_group_fd) < 0)
- RTE_LOG(INFO, EAL, "Error when clearing group for %s\n",
- dev_addr);
+ /* An VFIO group can have several devices attached. Just when there is
+ * no devices remaining should the group be closed.
+ */
+ if (--vfio_cfg.vfio_groups[vfio_group_fd].devices == 0) {
+
+ if (close(vfio_group_fd) < 0) {
+ RTE_LOG(INFO, EAL, "Error when closing vfio_group_fd for %s\n",
+ dev_addr);
+ return -1;
+ }
+
+ if (clear_group(vfio_group_fd) < 0) {
+ RTE_LOG(INFO, EAL, "Error when clearing group for %s\n",
+ dev_addr);
+ return -1;
+ }
+ }
return 0;
}
@@ -415,6 +430,7 @@
for (i = 0; i < VFIO_MAX_GROUPS; i++) {
vfio_cfg.vfio_groups[i].fd = -1;
vfio_cfg.vfio_groups[i].group_no = -1;
+ vfio_cfg.vfio_groups[i].devices = 0;
}
/* inform the user that we are probing for VFIO */
@@ -103,6 +103,7 @@ struct vfio_iommu_spapr_tce_remove {
struct vfio_group {
int group_no;
int fd;
+ int devices;
};
struct vfio_config {