diff mbox series

[v4,26/26] vhost: add VDUSE device stop

Message ID 20230601200812.672233-27-maxime.coquelin@redhat.com (mailing list archive)
State Superseded, archived
Delegated to: Maxime Coquelin
Headers
Series Add VDUSE support to Vhost library |

Checks

Context Check Description
ci/checkpatch success coding style OK
ci/loongarch-compilation success Compilation OK
ci/loongarch-unit-testing success Unit Testing PASS
ci/Intel-compilation success Compilation OK
ci/intel-Testing success Testing PASS
ci/intel-Functional success Functional PASS

Commit Message

Maxime Coquelin June 1, 2023, 8:08 p.m. UTC 
  This patch adds VDUSE device stop and cleanup of its
virtqueues.

Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
---
 doc/guides/rel_notes/release_23_07.rst |  6 +++
 lib/vhost/vduse.c                      | 72 +++++++++++++++++++++++---
 2 files changed, 70 insertions(+), 8 deletions(-)

Comments

Chenbo Xia June 5, 2023, 7:56 a.m. UTC | #1 
> -----Original Message-----
> From: Maxime Coquelin <maxime.coquelin@redhat.com>
> Sent: Friday, June 2, 2023 4:08 AM
> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>;
> david.marchand@redhat.com; mkp@redhat.com; fbl@redhat.com;
> jasowang@redhat.com; Liang, Cunming <cunming.liang@intel.com>; Xie, Yongji
> <xieyongji@bytedance.com>; echaudro@redhat.com; eperezma@redhat.com;
> amorenoz@redhat.com; lulu@redhat.com
> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
> Subject: [PATCH v4 26/26] vhost: add VDUSE device stop
> 
> This patch adds VDUSE device stop and cleanup of its
> virtqueues.
> 
> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
> ---
>  doc/guides/rel_notes/release_23_07.rst |  6 +++
>  lib/vhost/vduse.c                      | 72 +++++++++++++++++++++++---
>  2 files changed, 70 insertions(+), 8 deletions(-)
> 
> diff --git a/doc/guides/rel_notes/release_23_07.rst
> b/doc/guides/rel_notes/release_23_07.rst
> index 7034fb664c..6f43e8e633 100644
> --- a/doc/guides/rel_notes/release_23_07.rst
> +++ b/doc/guides/rel_notes/release_23_07.rst
> @@ -67,6 +67,12 @@ New Features
>    Introduced ``rte_vhost_driver_set_max_queue_num()`` to be able to limit
> the
>    maximum number of supported queue pairs, required for VDUSE support.
> 
> +* **Added VDUSE support into Vhost library.**
> +
> +  VDUSE aims at implementing vDPA devices in userspace. It can be used as
> an
> +  alternative to Vhost-user when using Vhost-vDPA, but also enable
> providing a
> +  virtio-net netdev to the host when using Virtio-vDPA driver.
> +
> 
>  Removed Items
>  -------------
> diff --git a/lib/vhost/vduse.c b/lib/vhost/vduse.c
> index 699cfed9e3..f421b1cf4c 100644
> --- a/lib/vhost/vduse.c
> +++ b/lib/vhost/vduse.c
> @@ -252,6 +252,44 @@ vduse_vring_setup(struct virtio_net *dev, unsigned
> int index)
>  	}
>  }
> 
> +static void
> +vduse_vring_cleanup(struct virtio_net *dev, unsigned int index)
> +{
> +	struct vhost_virtqueue *vq = dev->virtqueue[index];
> +	struct vduse_vq_eventfd vq_efd;
> +	int ret;
> +
> +	if (vq == dev->cvq && vq->kickfd >= 0) {
> +		fdset_del(&vduse.fdset, vq->kickfd);
> +		fdset_pipe_notify(&vduse.fdset);
> +	}
> +
> +	vq_efd.index = index;
> +	vq_efd.fd = VDUSE_EVENTFD_DEASSIGN;
> +
> +	ret = ioctl(dev->vduse_dev_fd, VDUSE_VQ_SETUP_KICKFD, &vq_efd);
> +	if (ret)
> +		VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to cleanup kickfd
> for VQ %u: %s\n",
> +				index, strerror(errno));
> +
> +	close(vq->kickfd);
> +	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
> +
> +	vring_invalidate(dev, vq);
> +
> +	rte_free(vq->batch_copy_elems);
> +	vq->batch_copy_elems = NULL;
> +
> +	rte_free(vq->shadow_used_split);
> +	vq->shadow_used_split = NULL;
> +
> +	vq->enabled = false;
> +	vq->ready = false;
> +	vq->size = 0;
> +	vq->last_used_idx = 0;
> +	vq->last_avail_idx = 0;
> +}
> +
>  static void
>  vduse_device_start(struct virtio_net *dev)
>  {
> @@ -304,6 +342,23 @@ vduse_device_start(struct virtio_net *dev)
>  	}
>  }
> 
> +static void
> +vduse_device_stop(struct virtio_net *dev)
> +{
> +	unsigned int i;
> +
> +	VHOST_LOG_CONFIG(dev->ifname, INFO, "Stopping device...\n");
> +
> +	vhost_destroy_device_notify(dev);
> +
> +	dev->flags &= ~VIRTIO_DEV_READY;
> +
> +	for (i = 0; i < dev->nr_vring; i++)
> +		vduse_vring_cleanup(dev, i);
> +
> +	vhost_user_iotlb_flush_all(dev);
> +}
> +
>  static void
>  vduse_events_handler(int fd, void *arg, int *remove __rte_unused)
>  {
> @@ -311,6 +366,7 @@ vduse_events_handler(int fd, void *arg, int *remove
> __rte_unused)
>  	struct vduse_dev_request req;
>  	struct vduse_dev_response resp;
>  	struct vhost_virtqueue *vq;
> +	uint8_t old_status;
>  	int ret;
> 
>  	memset(&resp, 0, sizeof(resp));
> @@ -339,10 +395,15 @@ vduse_events_handler(int fd, void *arg, int *remove
> __rte_unused)
>  	case VDUSE_SET_STATUS:
>  		VHOST_LOG_CONFIG(dev->ifname, INFO, "\tnew status: 0x%08x\n",
>  				req.s.status);
> +		old_status = dev->status;
>  		dev->status = req.s.status;
> 
> -		if (dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK)
> -			vduse_device_start(dev);
> +		if ((old_status ^ dev->status) &
> VIRTIO_DEVICE_STATUS_DRIVER_OK) {
> +			if (dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK)
> +				vduse_device_start(dev);
> +			else
> +				vduse_device_stop(dev);
> +		}
> 
>  		resp.result = VDUSE_REQ_RESULT_OK;
>  		break;
> @@ -560,12 +621,7 @@ vduse_device_destroy(const char *path)
>  	if (vid == RTE_MAX_VHOST_DEVICE)
>  		return -1;
> 
> -	if (dev->cvq && dev->cvq->kickfd >= 0) {
> -		fdset_del(&vduse.fdset, dev->cvq->kickfd);
> -		fdset_pipe_notify(&vduse.fdset);
> -		close(dev->cvq->kickfd);
> -		dev->cvq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
> -	}
> +	vduse_device_stop(dev);
> 
>  	fdset_del(&vduse.fdset, dev->vduse_dev_fd);
>  	fdset_pipe_notify(&vduse.fdset);
> --
> 2.40.1

Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
Maxime Coquelin June 6, 2023, 8:14 a.m. UTC | #2 
On 6/5/23 09:56, Xia, Chenbo wrote:
>> -----Original Message-----
>> From: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Sent: Friday, June 2, 2023 4:08 AM
>> To: dev@dpdk.org; Xia, Chenbo <chenbo.xia@intel.com>;
>> david.marchand@redhat.com; mkp@redhat.com; fbl@redhat.com;
>> jasowang@redhat.com; Liang, Cunming <cunming.liang@intel.com>; Xie, Yongji
>> <xieyongji@bytedance.com>; echaudro@redhat.com; eperezma@redhat.com;
>> amorenoz@redhat.com; lulu@redhat.com
>> Cc: Maxime Coquelin <maxime.coquelin@redhat.com>
>> Subject: [PATCH v4 26/26] vhost: add VDUSE device stop
>>
>> This patch adds VDUSE device stop and cleanup of its
>> virtqueues.
>>
>> Signed-off-by: Maxime Coquelin <maxime.coquelin@redhat.com>
>> ---
>>   doc/guides/rel_notes/release_23_07.rst |  6 +++
>>   lib/vhost/vduse.c                      | 72 +++++++++++++++++++++++---
>>   2 files changed, 70 insertions(+), 8 deletions(-)
>>
>> diff --git a/doc/guides/rel_notes/release_23_07.rst
>> b/doc/guides/rel_notes/release_23_07.rst
>> index 7034fb664c..6f43e8e633 100644
>> --- a/doc/guides/rel_notes/release_23_07.rst
>> +++ b/doc/guides/rel_notes/release_23_07.rst
>> @@ -67,6 +67,12 @@ New Features
>>     Introduced ``rte_vhost_driver_set_max_queue_num()`` to be able to limit
>> the
>>     maximum number of supported queue pairs, required for VDUSE support.
>>
>> +* **Added VDUSE support into Vhost library.**
>> +
>> +  VDUSE aims at implementing vDPA devices in userspace. It can be used as
>> an
>> +  alternative to Vhost-user when using Vhost-vDPA, but also enable
>> providing a
>> +  virtio-net netdev to the host when using Virtio-vDPA driver.
>> +
>>
>>   Removed Items
>>   -------------
>> diff --git a/lib/vhost/vduse.c b/lib/vhost/vduse.c
>> index 699cfed9e3..f421b1cf4c 100644
>> --- a/lib/vhost/vduse.c
>> +++ b/lib/vhost/vduse.c
>> @@ -252,6 +252,44 @@ vduse_vring_setup(struct virtio_net *dev, unsigned
>> int index)
>>   	}
>>   }
>>
>> +static void
>> +vduse_vring_cleanup(struct virtio_net *dev, unsigned int index)
>> +{
>> +	struct vhost_virtqueue *vq = dev->virtqueue[index];
>> +	struct vduse_vq_eventfd vq_efd;
>> +	int ret;
>> +
>> +	if (vq == dev->cvq && vq->kickfd >= 0) {
>> +		fdset_del(&vduse.fdset, vq->kickfd);
>> +		fdset_pipe_notify(&vduse.fdset);
>> +	}
>> +
>> +	vq_efd.index = index;
>> +	vq_efd.fd = VDUSE_EVENTFD_DEASSIGN;
>> +
>> +	ret = ioctl(dev->vduse_dev_fd, VDUSE_VQ_SETUP_KICKFD, &vq_efd);
>> +	if (ret)
>> +		VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to cleanup kickfd
>> for VQ %u: %s\n",
>> +				index, strerror(errno));
>> +
>> +	close(vq->kickfd);
>> +	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
>> +
>> +	vring_invalidate(dev, vq);
>> +
>> +	rte_free(vq->batch_copy_elems);
>> +	vq->batch_copy_elems = NULL;
>> +
>> +	rte_free(vq->shadow_used_split);
>> +	vq->shadow_used_split = NULL;
>> +
>> +	vq->enabled = false;
>> +	vq->ready = false;
>> +	vq->size = 0;
>> +	vq->last_used_idx = 0;
>> +	vq->last_avail_idx = 0;
>> +}
>> +
>>   static void
>>   vduse_device_start(struct virtio_net *dev)
>>   {
>> @@ -304,6 +342,23 @@ vduse_device_start(struct virtio_net *dev)
>>   	}
>>   }
>>
>> +static void
>> +vduse_device_stop(struct virtio_net *dev)
>> +{
>> +	unsigned int i;
>> +
>> +	VHOST_LOG_CONFIG(dev->ifname, INFO, "Stopping device...\n");
>> +
>> +	vhost_destroy_device_notify(dev);
>> +
>> +	dev->flags &= ~VIRTIO_DEV_READY;
>> +
>> +	for (i = 0; i < dev->nr_vring; i++)
>> +		vduse_vring_cleanup(dev, i);
>> +
>> +	vhost_user_iotlb_flush_all(dev);
>> +}
>> +
>>   static void
>>   vduse_events_handler(int fd, void *arg, int *remove __rte_unused)
>>   {
>> @@ -311,6 +366,7 @@ vduse_events_handler(int fd, void *arg, int *remove
>> __rte_unused)
>>   	struct vduse_dev_request req;
>>   	struct vduse_dev_response resp;
>>   	struct vhost_virtqueue *vq;
>> +	uint8_t old_status;
>>   	int ret;
>>
>>   	memset(&resp, 0, sizeof(resp));
>> @@ -339,10 +395,15 @@ vduse_events_handler(int fd, void *arg, int *remove
>> __rte_unused)
>>   	case VDUSE_SET_STATUS:
>>   		VHOST_LOG_CONFIG(dev->ifname, INFO, "\tnew status: 0x%08x\n",
>>   				req.s.status);
>> +		old_status = dev->status;
>>   		dev->status = req.s.status;
>>
>> -		if (dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK)
>> -			vduse_device_start(dev);
>> +		if ((old_status ^ dev->status) &
>> VIRTIO_DEVICE_STATUS_DRIVER_OK) {
>> +			if (dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK)
>> +				vduse_device_start(dev);
>> +			else
>> +				vduse_device_stop(dev);
>> +		}
>>
>>   		resp.result = VDUSE_REQ_RESULT_OK;
>>   		break;
>> @@ -560,12 +621,7 @@ vduse_device_destroy(const char *path)
>>   	if (vid == RTE_MAX_VHOST_DEVICE)
>>   		return -1;
>>
>> -	if (dev->cvq && dev->cvq->kickfd >= 0) {
>> -		fdset_del(&vduse.fdset, dev->cvq->kickfd);
>> -		fdset_pipe_notify(&vduse.fdset);
>> -		close(dev->cvq->kickfd);
>> -		dev->cvq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
>> -	}
>> +	vduse_device_stop(dev);
>>
>>   	fdset_del(&vduse.fdset, dev->vduse_dev_fd);
>>   	fdset_pipe_notify(&vduse.fdset);
>> --
>> 2.40.1
> 
> Reviewed-by: Chenbo Xia <chenbo.xia@intel.com>
> 

Thanks Chenbo, I just fixed an issue when testing with OVS that impact
both this patch and the one adding VDUSE device startup.

So your R-by on these patches will be removed for both these patches in
v5.

Maxime
diff mbox series

Patch

diff --git a/doc/guides/rel_notes/release_23_07.rst b/doc/guides/rel_notes/release_23_07.rst
index 7034fb664c..6f43e8e633 100644
--- a/doc/guides/rel_notes/release_23_07.rst
+++ b/doc/guides/rel_notes/release_23_07.rst
@@ -67,6 +67,12 @@  New Features
   Introduced ``rte_vhost_driver_set_max_queue_num()`` to be able to limit the
   maximum number of supported queue pairs, required for VDUSE support.
 
+* **Added VDUSE support into Vhost library.**
+
+  VDUSE aims at implementing vDPA devices in userspace. It can be used as an
+  alternative to Vhost-user when using Vhost-vDPA, but also enable providing a
+  virtio-net netdev to the host when using Virtio-vDPA driver.
+
 
 Removed Items
 -------------
diff --git a/lib/vhost/vduse.c b/lib/vhost/vduse.c
index 699cfed9e3..f421b1cf4c 100644
--- a/lib/vhost/vduse.c
+++ b/lib/vhost/vduse.c
@@ -252,6 +252,44 @@  vduse_vring_setup(struct virtio_net *dev, unsigned int index)
 	}
 }
 
+static void
+vduse_vring_cleanup(struct virtio_net *dev, unsigned int index)
+{
+	struct vhost_virtqueue *vq = dev->virtqueue[index];
+	struct vduse_vq_eventfd vq_efd;
+	int ret;
+
+	if (vq == dev->cvq && vq->kickfd >= 0) {
+		fdset_del(&vduse.fdset, vq->kickfd);
+		fdset_pipe_notify(&vduse.fdset);
+	}
+
+	vq_efd.index = index;
+	vq_efd.fd = VDUSE_EVENTFD_DEASSIGN;
+
+	ret = ioctl(dev->vduse_dev_fd, VDUSE_VQ_SETUP_KICKFD, &vq_efd);
+	if (ret)
+		VHOST_LOG_CONFIG(dev->ifname, ERR, "Failed to cleanup kickfd for VQ %u: %s\n",
+				index, strerror(errno));
+
+	close(vq->kickfd);
+	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
+
+	vring_invalidate(dev, vq);
+
+	rte_free(vq->batch_copy_elems);
+	vq->batch_copy_elems = NULL;
+
+	rte_free(vq->shadow_used_split);
+	vq->shadow_used_split = NULL;
+
+	vq->enabled = false;
+	vq->ready = false;
+	vq->size = 0;
+	vq->last_used_idx = 0;
+	vq->last_avail_idx = 0;
+}
+
 static void
 vduse_device_start(struct virtio_net *dev)
 {
@@ -304,6 +342,23 @@  vduse_device_start(struct virtio_net *dev)
 	}
 }
 
+static void
+vduse_device_stop(struct virtio_net *dev)
+{
+	unsigned int i;
+
+	VHOST_LOG_CONFIG(dev->ifname, INFO, "Stopping device...\n");
+
+	vhost_destroy_device_notify(dev);
+
+	dev->flags &= ~VIRTIO_DEV_READY;
+
+	for (i = 0; i < dev->nr_vring; i++)
+		vduse_vring_cleanup(dev, i);
+
+	vhost_user_iotlb_flush_all(dev);
+}
+
 static void
 vduse_events_handler(int fd, void *arg, int *remove __rte_unused)
 {
@@ -311,6 +366,7 @@  vduse_events_handler(int fd, void *arg, int *remove __rte_unused)
 	struct vduse_dev_request req;
 	struct vduse_dev_response resp;
 	struct vhost_virtqueue *vq;
+	uint8_t old_status;
 	int ret;
 
 	memset(&resp, 0, sizeof(resp));
@@ -339,10 +395,15 @@  vduse_events_handler(int fd, void *arg, int *remove __rte_unused)
 	case VDUSE_SET_STATUS:
 		VHOST_LOG_CONFIG(dev->ifname, INFO, "\tnew status: 0x%08x\n",
 				req.s.status);
+		old_status = dev->status;
 		dev->status = req.s.status;
 
-		if (dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK)
-			vduse_device_start(dev);
+		if ((old_status ^ dev->status) & VIRTIO_DEVICE_STATUS_DRIVER_OK) {
+			if (dev->status & VIRTIO_DEVICE_STATUS_DRIVER_OK)
+				vduse_device_start(dev);
+			else
+				vduse_device_stop(dev);
+		}
 
 		resp.result = VDUSE_REQ_RESULT_OK;
 		break;
@@ -560,12 +621,7 @@  vduse_device_destroy(const char *path)
 	if (vid == RTE_MAX_VHOST_DEVICE)
 		return -1;
 
-	if (dev->cvq && dev->cvq->kickfd >= 0) {
-		fdset_del(&vduse.fdset, dev->cvq->kickfd);
-		fdset_pipe_notify(&vduse.fdset);
-		close(dev->cvq->kickfd);
-		dev->cvq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
-	}
+	vduse_device_stop(dev);
 
 	fdset_del(&vduse.fdset, dev->vduse_dev_fd);
 	fdset_pipe_notify(&vduse.fdset);