diff mbox series

[v8,12/13] vdpa/ifc: add interrupt handling for config space

Message ID 1652876035-70513-13-git-send-email-andy.pei@intel.com (mailing list archive)
State Superseded
Delegated to: Maxime Coquelin
Headers show
Series add virtio_blk device support to vdpa/ifc | expand

Checks

Context Check Description
ci/checkpatch warning coding style issues

Commit Message

Andy Pei May 18, 2022, 12:13 p.m. UTC
Create a thread to poll and relay config space change interrupt.
Use VHOST_USER_SLAVE_CONFIG_CHANGE_MSG to inform QEMU.

Signed-off-by: Andy Pei <andy.pei@intel.com>
---
 drivers/vdpa/ifc/ifcvf_vdpa.c | 118 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 117 insertions(+), 1 deletion(-)

Comments

Xia, Chenbo May 23, 2022, 7:54 a.m. UTC | #1
> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Wednesday, May 18, 2022 8:14 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; maxime.coquelin@redhat.com; Cao,
> Gang <gang.cao@intel.com>; Liu, Changpeng <changpeng.liu@intel.com>; Xu,
> Rosen <rosen.xu@intel.com>; Xiao, QimaiX <qimaix.xiao@intel.com>
> Subject: [PATCH v8 12/13] vdpa/ifc: add interrupt handling for config
> space
> 
> Create a thread to poll and relay config space change interrupt.
> Use VHOST_USER_SLAVE_CONFIG_CHANGE_MSG to inform QEMU.
> 
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
>  drivers/vdpa/ifc/ifcvf_vdpa.c | 118
> +++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 117 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 376a1af..8a49622 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -53,7 +53,9 @@ struct ifcvf_internal {
>  	int vfio_group_fd;
>  	int vfio_dev_fd;
>  	pthread_t tid;	/* thread for notify relay */
> +	pthread_t intr_tid; /* thread for config space change interrupt
> relay */
>  	int epfd;
> +	int csc_epfd;
>  	int vid;
>  	struct rte_vdpa_device *vdev;
>  	uint16_t max_queues;
> @@ -566,6 +568,111 @@ struct rte_vdpa_dev_info {
>  	return 0;
>  }
> 
> +static void
> +virtio_interrupt_handler(struct ifcvf_internal *internal)
> +{
> +	int vid = internal->vid;
> +	int ret;
> +
> +	ret = rte_vhost_slave_config_change(vid, 1);
> +	if (ret)
> +		DRV_LOG(ERR, "failed to notify the guest about configuration
> space change.");
> +}
> +
> +static void *
> +intr_relay(void *arg)
> +{
> +	struct ifcvf_internal *internal = (struct ifcvf_internal *)arg;
> +	struct epoll_event csc_event;
> +	struct epoll_event ev;
> +	uint64_t buf;
> +	int nbytes;
> +	int csc_epfd, csc_val = 0;
> +
> +	csc_epfd = epoll_create(1);
> +	if (csc_epfd < 0) {
> +		DRV_LOG(ERR, "failed to create epoll for config space
> change.");
> +		return NULL;
> +	}
> +
> +	ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
> +	ev.data.fd = rte_intr_fd_get(internal->pdev->intr_handle);
> +	if (epoll_ctl(csc_epfd, EPOLL_CTL_ADD,
> +		rte_intr_fd_get(internal->pdev->intr_handle), &ev) < 0) {
> +		DRV_LOG(ERR, "epoll add error: %s", strerror(errno));
> +		return NULL;
> +	}
> +
> +	internal->csc_epfd = csc_epfd;
> +
> +	for (;;) {
> +		csc_val = epoll_wait(csc_epfd, &csc_event, 1, -1);
> +		if (csc_val < 0) {
> +			if (errno == EINTR)
> +				continue;
> +			DRV_LOG(ERR, "epoll_wait return fail.");
> +			return NULL;
> +		} else if (csc_val == 0) {
> +			continue;
> +		} else {
> +			/* csc_val > 0 */
> +			nbytes = read(csc_event.data.fd, &buf, 8);
> +			if (nbytes < 0) {
> +				if (errno == EINTR ||
> +				    errno == EWOULDBLOCK ||
> +				    errno == EAGAIN)
> +					continue;
> +				DRV_LOG(ERR, "Error reading from file
> descriptor %d: %s\n",
> +					csc_event.data.fd,
> +					strerror(errno));
> +				return NULL;
> +			} else if (nbytes == 0) {
> +				DRV_LOG(ERR, "Read nothing from file
> descriptor %d\n",
> +					csc_event.data.fd);
> +				continue;
> +			} else {
> +				virtio_interrupt_handler(internal);
> +			}
> +		}
> +	}
> +
> +	return NULL;
> +}

I think we should not assume unset_intr_relay will help us close epfd when
Error happens, so just close it when there's some error.

> +
> +static int
> +setup_intr_relay(struct ifcvf_internal *internal)
> +{
> +	char name[THREAD_NAME_LEN];
> +	int ret;
> +
> +	snprintf(name, sizeof(name), "ifc-intr-%d", internal->vid);
> +	ret = rte_ctrl_thread_create(&internal->intr_tid, name, NULL,
> +				     intr_relay, (void *)internal);
> +	if (ret) {
> +		DRV_LOG(ERR, "failed to create notify relay pthread.");
> +		return -1;
> +	}
> +	return 0;
> +}
> +
> +static int
> +unset_intr_relay(struct ifcvf_internal *internal)
> +{
> +	void *status;
> +
> +	if (internal->intr_tid) {
> +		pthread_cancel(internal->intr_tid);
> +		pthread_join(internal->intr_tid, &status);
> +	}
> +	internal->intr_tid = 0;
> +
> +	if (internal->csc_epfd >= 0)
> +		close(internal->csc_epfd);
> +	internal->csc_epfd = -1;
> +
> +	return 0;
> +}

It will always return 0, so return type should be void

> +
>  static int
>  update_datapath(struct ifcvf_internal *internal)
>  {
> @@ -592,10 +699,16 @@ struct rte_vdpa_dev_info {
>  		if (ret)
>  			goto err;
> 
> +		ret = setup_intr_relay(internal);
> +		if (ret)
> +			goto err;
> +
>  		rte_atomic32_set(&internal->running, 1);
>  	} else if (rte_atomic32_read(&internal->running) &&
>  		   (!rte_atomic32_read(&internal->started) ||
>  		    !rte_atomic32_read(&internal->dev_attached))) {
> +		ret = unset_intr_relay(internal);

This will be changed accordingly.

Thanks,
Chenbo

> +
>  		ret = unset_notify_relay(internal);
>  		if (ret)
>  			goto err;
> @@ -812,7 +925,7 @@ struct rte_vdpa_dev_info {
>  		if (nfds < 0) {
>  			if (errno == EINTR)
>  				continue;
> -			DRV_LOG(ERR, "epoll_wait return fail\n");
> +			DRV_LOG(ERR, "epoll_wait return fail.");
>  			return NULL;
>  		}
> 
> @@ -888,6 +1001,9 @@ struct rte_vdpa_dev_info {
>  	/* stop the direct IO data path */
>  	unset_notify_relay(internal);
>  	vdpa_ifcvf_stop(internal);
> +
> +	unset_intr_relay(internal);
> +
>  	vdpa_disable_vfio_intr(internal);
> 
>  	ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);
> --
> 1.8.3.1
diff mbox series

Patch

diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
index 376a1af..8a49622 100644
--- a/drivers/vdpa/ifc/ifcvf_vdpa.c
+++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
@@ -53,7 +53,9 @@  struct ifcvf_internal {
 	int vfio_group_fd;
 	int vfio_dev_fd;
 	pthread_t tid;	/* thread for notify relay */
+	pthread_t intr_tid; /* thread for config space change interrupt relay */
 	int epfd;
+	int csc_epfd;
 	int vid;
 	struct rte_vdpa_device *vdev;
 	uint16_t max_queues;
@@ -566,6 +568,111 @@  struct rte_vdpa_dev_info {
 	return 0;
 }
 
+static void
+virtio_interrupt_handler(struct ifcvf_internal *internal)
+{
+	int vid = internal->vid;
+	int ret;
+
+	ret = rte_vhost_slave_config_change(vid, 1);
+	if (ret)
+		DRV_LOG(ERR, "failed to notify the guest about configuration space change.");
+}
+
+static void *
+intr_relay(void *arg)
+{
+	struct ifcvf_internal *internal = (struct ifcvf_internal *)arg;
+	struct epoll_event csc_event;
+	struct epoll_event ev;
+	uint64_t buf;
+	int nbytes;
+	int csc_epfd, csc_val = 0;
+
+	csc_epfd = epoll_create(1);
+	if (csc_epfd < 0) {
+		DRV_LOG(ERR, "failed to create epoll for config space change.");
+		return NULL;
+	}
+
+	ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
+	ev.data.fd = rte_intr_fd_get(internal->pdev->intr_handle);
+	if (epoll_ctl(csc_epfd, EPOLL_CTL_ADD,
+		rte_intr_fd_get(internal->pdev->intr_handle), &ev) < 0) {
+		DRV_LOG(ERR, "epoll add error: %s", strerror(errno));
+		return NULL;
+	}
+
+	internal->csc_epfd = csc_epfd;
+
+	for (;;) {
+		csc_val = epoll_wait(csc_epfd, &csc_event, 1, -1);
+		if (csc_val < 0) {
+			if (errno == EINTR)
+				continue;
+			DRV_LOG(ERR, "epoll_wait return fail.");
+			return NULL;
+		} else if (csc_val == 0) {
+			continue;
+		} else {
+			/* csc_val > 0 */
+			nbytes = read(csc_event.data.fd, &buf, 8);
+			if (nbytes < 0) {
+				if (errno == EINTR ||
+				    errno == EWOULDBLOCK ||
+				    errno == EAGAIN)
+					continue;
+				DRV_LOG(ERR, "Error reading from file descriptor %d: %s\n",
+					csc_event.data.fd,
+					strerror(errno));
+				return NULL;
+			} else if (nbytes == 0) {
+				DRV_LOG(ERR, "Read nothing from file descriptor %d\n",
+					csc_event.data.fd);
+				continue;
+			} else {
+				virtio_interrupt_handler(internal);
+			}
+		}
+	}
+
+	return NULL;
+}
+
+static int
+setup_intr_relay(struct ifcvf_internal *internal)
+{
+	char name[THREAD_NAME_LEN];
+	int ret;
+
+	snprintf(name, sizeof(name), "ifc-intr-%d", internal->vid);
+	ret = rte_ctrl_thread_create(&internal->intr_tid, name, NULL,
+				     intr_relay, (void *)internal);
+	if (ret) {
+		DRV_LOG(ERR, "failed to create notify relay pthread.");
+		return -1;
+	}
+	return 0;
+}
+
+static int
+unset_intr_relay(struct ifcvf_internal *internal)
+{
+	void *status;
+
+	if (internal->intr_tid) {
+		pthread_cancel(internal->intr_tid);
+		pthread_join(internal->intr_tid, &status);
+	}
+	internal->intr_tid = 0;
+
+	if (internal->csc_epfd >= 0)
+		close(internal->csc_epfd);
+	internal->csc_epfd = -1;
+
+	return 0;
+}
+
 static int
 update_datapath(struct ifcvf_internal *internal)
 {
@@ -592,10 +699,16 @@  struct rte_vdpa_dev_info {
 		if (ret)
 			goto err;
 
+		ret = setup_intr_relay(internal);
+		if (ret)
+			goto err;
+
 		rte_atomic32_set(&internal->running, 1);
 	} else if (rte_atomic32_read(&internal->running) &&
 		   (!rte_atomic32_read(&internal->started) ||
 		    !rte_atomic32_read(&internal->dev_attached))) {
+		ret = unset_intr_relay(internal);
+
 		ret = unset_notify_relay(internal);
 		if (ret)
 			goto err;
@@ -812,7 +925,7 @@  struct rte_vdpa_dev_info {
 		if (nfds < 0) {
 			if (errno == EINTR)
 				continue;
-			DRV_LOG(ERR, "epoll_wait return fail\n");
+			DRV_LOG(ERR, "epoll_wait return fail.");
 			return NULL;
 		}
 
@@ -888,6 +1001,9 @@  struct rte_vdpa_dev_info {
 	/* stop the direct IO data path */
 	unset_notify_relay(internal);
 	vdpa_ifcvf_stop(internal);
+
+	unset_intr_relay(internal);
+
 	vdpa_disable_vfio_intr(internal);
 
 	ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);