[v8,12/13] vdpa/ifc: add interrupt handling for config space
Checks
Commit Message
Create a thread to poll and relay config space change interrupt.
Use VHOST_USER_SLAVE_CONFIG_CHANGE_MSG to inform QEMU.
Signed-off-by: Andy Pei <andy.pei@intel.com>
---
drivers/vdpa/ifc/ifcvf_vdpa.c | 118 +++++++++++++++++++++++++++++++++++++++++-
1 file changed, 117 insertions(+), 1 deletion(-)
Comments
> -----Original Message-----
> From: Pei, Andy <andy.pei@intel.com>
> Sent: Wednesday, May 18, 2022 8:14 PM
> To: dev@dpdk.org
> Cc: Xia, Chenbo <chenbo.xia@intel.com>; maxime.coquelin@redhat.com; Cao,
> Gang <gang.cao@intel.com>; Liu, Changpeng <changpeng.liu@intel.com>; Xu,
> Rosen <rosen.xu@intel.com>; Xiao, QimaiX <qimaix.xiao@intel.com>
> Subject: [PATCH v8 12/13] vdpa/ifc: add interrupt handling for config
> space
>
> Create a thread to poll and relay config space change interrupt.
> Use VHOST_USER_SLAVE_CONFIG_CHANGE_MSG to inform QEMU.
>
> Signed-off-by: Andy Pei <andy.pei@intel.com>
> ---
> drivers/vdpa/ifc/ifcvf_vdpa.c | 118
> +++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 117 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/vdpa/ifc/ifcvf_vdpa.c b/drivers/vdpa/ifc/ifcvf_vdpa.c
> index 376a1af..8a49622 100644
> --- a/drivers/vdpa/ifc/ifcvf_vdpa.c
> +++ b/drivers/vdpa/ifc/ifcvf_vdpa.c
> @@ -53,7 +53,9 @@ struct ifcvf_internal {
> int vfio_group_fd;
> int vfio_dev_fd;
> pthread_t tid; /* thread for notify relay */
> + pthread_t intr_tid; /* thread for config space change interrupt
> relay */
> int epfd;
> + int csc_epfd;
> int vid;
> struct rte_vdpa_device *vdev;
> uint16_t max_queues;
> @@ -566,6 +568,111 @@ struct rte_vdpa_dev_info {
> return 0;
> }
>
> +static void
> +virtio_interrupt_handler(struct ifcvf_internal *internal)
> +{
> + int vid = internal->vid;
> + int ret;
> +
> + ret = rte_vhost_slave_config_change(vid, 1);
> + if (ret)
> + DRV_LOG(ERR, "failed to notify the guest about configuration
> space change.");
> +}
> +
> +static void *
> +intr_relay(void *arg)
> +{
> + struct ifcvf_internal *internal = (struct ifcvf_internal *)arg;
> + struct epoll_event csc_event;
> + struct epoll_event ev;
> + uint64_t buf;
> + int nbytes;
> + int csc_epfd, csc_val = 0;
> +
> + csc_epfd = epoll_create(1);
> + if (csc_epfd < 0) {
> + DRV_LOG(ERR, "failed to create epoll for config space
> change.");
> + return NULL;
> + }
> +
> + ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
> + ev.data.fd = rte_intr_fd_get(internal->pdev->intr_handle);
> + if (epoll_ctl(csc_epfd, EPOLL_CTL_ADD,
> + rte_intr_fd_get(internal->pdev->intr_handle), &ev) < 0) {
> + DRV_LOG(ERR, "epoll add error: %s", strerror(errno));
> + return NULL;
> + }
> +
> + internal->csc_epfd = csc_epfd;
> +
> + for (;;) {
> + csc_val = epoll_wait(csc_epfd, &csc_event, 1, -1);
> + if (csc_val < 0) {
> + if (errno == EINTR)
> + continue;
> + DRV_LOG(ERR, "epoll_wait return fail.");
> + return NULL;
> + } else if (csc_val == 0) {
> + continue;
> + } else {
> + /* csc_val > 0 */
> + nbytes = read(csc_event.data.fd, &buf, 8);
> + if (nbytes < 0) {
> + if (errno == EINTR ||
> + errno == EWOULDBLOCK ||
> + errno == EAGAIN)
> + continue;
> + DRV_LOG(ERR, "Error reading from file
> descriptor %d: %s\n",
> + csc_event.data.fd,
> + strerror(errno));
> + return NULL;
> + } else if (nbytes == 0) {
> + DRV_LOG(ERR, "Read nothing from file
> descriptor %d\n",
> + csc_event.data.fd);
> + continue;
> + } else {
> + virtio_interrupt_handler(internal);
> + }
> + }
> + }
> +
> + return NULL;
> +}
I think we should not assume unset_intr_relay will help us close epfd when
Error happens, so just close it when there's some error.
> +
> +static int
> +setup_intr_relay(struct ifcvf_internal *internal)
> +{
> + char name[THREAD_NAME_LEN];
> + int ret;
> +
> + snprintf(name, sizeof(name), "ifc-intr-%d", internal->vid);
> + ret = rte_ctrl_thread_create(&internal->intr_tid, name, NULL,
> + intr_relay, (void *)internal);
> + if (ret) {
> + DRV_LOG(ERR, "failed to create notify relay pthread.");
> + return -1;
> + }
> + return 0;
> +}
> +
> +static int
> +unset_intr_relay(struct ifcvf_internal *internal)
> +{
> + void *status;
> +
> + if (internal->intr_tid) {
> + pthread_cancel(internal->intr_tid);
> + pthread_join(internal->intr_tid, &status);
> + }
> + internal->intr_tid = 0;
> +
> + if (internal->csc_epfd >= 0)
> + close(internal->csc_epfd);
> + internal->csc_epfd = -1;
> +
> + return 0;
> +}
It will always return 0, so return type should be void
> +
> static int
> update_datapath(struct ifcvf_internal *internal)
> {
> @@ -592,10 +699,16 @@ struct rte_vdpa_dev_info {
> if (ret)
> goto err;
>
> + ret = setup_intr_relay(internal);
> + if (ret)
> + goto err;
> +
> rte_atomic32_set(&internal->running, 1);
> } else if (rte_atomic32_read(&internal->running) &&
> (!rte_atomic32_read(&internal->started) ||
> !rte_atomic32_read(&internal->dev_attached))) {
> + ret = unset_intr_relay(internal);
This will be changed accordingly.
Thanks,
Chenbo
> +
> ret = unset_notify_relay(internal);
> if (ret)
> goto err;
> @@ -812,7 +925,7 @@ struct rte_vdpa_dev_info {
> if (nfds < 0) {
> if (errno == EINTR)
> continue;
> - DRV_LOG(ERR, "epoll_wait return fail\n");
> + DRV_LOG(ERR, "epoll_wait return fail.");
> return NULL;
> }
>
> @@ -888,6 +1001,9 @@ struct rte_vdpa_dev_info {
> /* stop the direct IO data path */
> unset_notify_relay(internal);
> vdpa_ifcvf_stop(internal);
> +
> + unset_intr_relay(internal);
> +
> vdpa_disable_vfio_intr(internal);
>
> ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);
> --
> 1.8.3.1
@@ -53,7 +53,9 @@ struct ifcvf_internal {
int vfio_group_fd;
int vfio_dev_fd;
pthread_t tid; /* thread for notify relay */
+ pthread_t intr_tid; /* thread for config space change interrupt relay */
int epfd;
+ int csc_epfd;
int vid;
struct rte_vdpa_device *vdev;
uint16_t max_queues;
@@ -566,6 +568,111 @@ struct rte_vdpa_dev_info {
return 0;
}
+static void
+virtio_interrupt_handler(struct ifcvf_internal *internal)
+{
+ int vid = internal->vid;
+ int ret;
+
+ ret = rte_vhost_slave_config_change(vid, 1);
+ if (ret)
+ DRV_LOG(ERR, "failed to notify the guest about configuration space change.");
+}
+
+static void *
+intr_relay(void *arg)
+{
+ struct ifcvf_internal *internal = (struct ifcvf_internal *)arg;
+ struct epoll_event csc_event;
+ struct epoll_event ev;
+ uint64_t buf;
+ int nbytes;
+ int csc_epfd, csc_val = 0;
+
+ csc_epfd = epoll_create(1);
+ if (csc_epfd < 0) {
+ DRV_LOG(ERR, "failed to create epoll for config space change.");
+ return NULL;
+ }
+
+ ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
+ ev.data.fd = rte_intr_fd_get(internal->pdev->intr_handle);
+ if (epoll_ctl(csc_epfd, EPOLL_CTL_ADD,
+ rte_intr_fd_get(internal->pdev->intr_handle), &ev) < 0) {
+ DRV_LOG(ERR, "epoll add error: %s", strerror(errno));
+ return NULL;
+ }
+
+ internal->csc_epfd = csc_epfd;
+
+ for (;;) {
+ csc_val = epoll_wait(csc_epfd, &csc_event, 1, -1);
+ if (csc_val < 0) {
+ if (errno == EINTR)
+ continue;
+ DRV_LOG(ERR, "epoll_wait return fail.");
+ return NULL;
+ } else if (csc_val == 0) {
+ continue;
+ } else {
+ /* csc_val > 0 */
+ nbytes = read(csc_event.data.fd, &buf, 8);
+ if (nbytes < 0) {
+ if (errno == EINTR ||
+ errno == EWOULDBLOCK ||
+ errno == EAGAIN)
+ continue;
+ DRV_LOG(ERR, "Error reading from file descriptor %d: %s\n",
+ csc_event.data.fd,
+ strerror(errno));
+ return NULL;
+ } else if (nbytes == 0) {
+ DRV_LOG(ERR, "Read nothing from file descriptor %d\n",
+ csc_event.data.fd);
+ continue;
+ } else {
+ virtio_interrupt_handler(internal);
+ }
+ }
+ }
+
+ return NULL;
+}
+
+static int
+setup_intr_relay(struct ifcvf_internal *internal)
+{
+ char name[THREAD_NAME_LEN];
+ int ret;
+
+ snprintf(name, sizeof(name), "ifc-intr-%d", internal->vid);
+ ret = rte_ctrl_thread_create(&internal->intr_tid, name, NULL,
+ intr_relay, (void *)internal);
+ if (ret) {
+ DRV_LOG(ERR, "failed to create notify relay pthread.");
+ return -1;
+ }
+ return 0;
+}
+
+static int
+unset_intr_relay(struct ifcvf_internal *internal)
+{
+ void *status;
+
+ if (internal->intr_tid) {
+ pthread_cancel(internal->intr_tid);
+ pthread_join(internal->intr_tid, &status);
+ }
+ internal->intr_tid = 0;
+
+ if (internal->csc_epfd >= 0)
+ close(internal->csc_epfd);
+ internal->csc_epfd = -1;
+
+ return 0;
+}
+
static int
update_datapath(struct ifcvf_internal *internal)
{
@@ -592,10 +699,16 @@ struct rte_vdpa_dev_info {
if (ret)
goto err;
+ ret = setup_intr_relay(internal);
+ if (ret)
+ goto err;
+
rte_atomic32_set(&internal->running, 1);
} else if (rte_atomic32_read(&internal->running) &&
(!rte_atomic32_read(&internal->started) ||
!rte_atomic32_read(&internal->dev_attached))) {
+ ret = unset_intr_relay(internal);
+
ret = unset_notify_relay(internal);
if (ret)
goto err;
@@ -812,7 +925,7 @@ struct rte_vdpa_dev_info {
if (nfds < 0) {
if (errno == EINTR)
continue;
- DRV_LOG(ERR, "epoll_wait return fail\n");
+ DRV_LOG(ERR, "epoll_wait return fail.");
return NULL;
}
@@ -888,6 +1001,9 @@ struct rte_vdpa_dev_info {
/* stop the direct IO data path */
unset_notify_relay(internal);
vdpa_ifcvf_stop(internal);
+
+ unset_intr_relay(internal);
+
vdpa_disable_vfio_intr(internal);
ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);