[dpdk-dev,v3,3/4] net/virtio-user: add support for server mode
Checks
Commit Message
virtio-user adds support for server mode in this patch.
Virtio-user with server mode creates socket file and then starts to wait
for the first connection from vhost user with client mode in blocking mode.
Server mode virtio-user supports many times' vhost reconnections with
the same configurations.
Signed-off-by: Zhiyong Yang <zhiyong.yang@intel.com>
---
drivers/net/virtio/virtio_user/vhost_user.c | 96 ++++++++++++++++++++++--
drivers/net/virtio/virtio_user/virtio_user_dev.c | 47 ++++++++----
drivers/net/virtio/virtio_user/virtio_user_dev.h | 11 +++
drivers/net/virtio/virtio_user_ethdev.c | 83 +++++++++++++++++++-
4 files changed, 215 insertions(+), 22 deletions(-)
Comments
Hi Zhiyong,
Triggered by the community discussion of thread creation in a library
(http://dpdk.org/dev/patchwork/patch/36579/), we can think about if
possible to avoid the pthread creation here.
Just a quick think, it could be feasible. The key idea is to properly
set up the LSC interrupt for a server mode virtio-user.
- If the virtio-user device is just probed, we can setup LSC interrupt
by registering the listen fd to interrupt thread. Whenever a vhost-user
connection is coming, we will get a chance to read status, at that time,
we could accept the connection, and report the status is up.
- And after the connection is setup, we setup LSC interrupt by
registering the connection fd to interrupt thread; if the connection is
broken, we also get a change to read status, in which we can switch to
the above setting.
How do you think?
Thanks,
Jianfeng
On 3/21/2018 11:03 AM, zhiyong.yang@intel.com wrote:
> virtio-user adds support for server mode in this patch.
>
> Virtio-user with server mode creates socket file and then starts to wait
> for the first connection from vhost user with client mode in blocking mode.
>
> Server mode virtio-user supports many times' vhost reconnections with
> the same configurations.
>
> Signed-off-by: Zhiyong Yang <zhiyong.yang@intel.com>
> ---
> drivers/net/virtio/virtio_user/vhost_user.c | 96 ++++++++++++++++++++++--
> drivers/net/virtio/virtio_user/virtio_user_dev.c | 47 ++++++++----
> drivers/net/virtio/virtio_user/virtio_user_dev.h | 11 +++
> drivers/net/virtio/virtio_user_ethdev.c | 83 +++++++++++++++++++-
> 4 files changed, 215 insertions(+), 22 deletions(-)
>
> diff --git a/drivers/net/virtio/virtio_user/vhost_user.c b/drivers/net/virtio/virtio_user/vhost_user.c
> index 91c6449bb..90f4fed31 100644
> --- a/drivers/net/virtio/virtio_user/vhost_user.c
> +++ b/drivers/net/virtio/virtio_user/vhost_user.c
> @@ -378,6 +378,65 @@ vhost_user_sock(struct virtio_user_dev *dev,
> return 0;
> }
>
> +static void
> +virtio_user_set_block(int fd, bool enabled)
> +{
> + int f;
> +
> + f = fcntl(fd, F_GETFL);
> + if (enabled)
> + fcntl(fd, F_SETFL, f & ~O_NONBLOCK);
> + else
> + fcntl(fd, F_SETFL, f | O_NONBLOCK);
> +}
> +
> +#define MAX_VIRTIO_USER_BACKLOG 128
> +static int
> +virtio_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un)
> +{
> + int ret;
> + int fd = dev->listenfd;
> + int connectfd;
> +
> + ret = bind(fd, (struct sockaddr *)un, sizeof(*un));
> + if (ret < 0) {
> + PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again\n",
> + dev->path, strerror(errno));
> + goto err;
> + }
> + ret = listen(fd, MAX_VIRTIO_USER_BACKLOG);
> + if (ret < 0)
> + goto err;
> +
> + virtio_user_set_block(fd, true);
> + PMD_DRV_LOG(NOTICE, "virtio user server mode is waiting for connection from vhost user.");
> + while (1) {
> + connectfd = accept(fd, NULL, NULL);
> + if (connectfd >= 0) {
> + dev->connected = true;
> + break;
> + }
> + }
> +
> + dev->vhostfd = connectfd;
> + virtio_user_set_block(connectfd, true);
> +
> + return 0;
> +err:
> + close(dev->epoll_fd);
> + close(dev->listenfd);
> + return -1;
> +}
> +
> +static __attribute__((noreturn)) void *
> +event_dispatch(void *arg)
> +{
> + struct virtio_user_dev *dev = arg;
> +
> + while (1)
> + rte_epoll_wait(dev->epoll_fd, &dev->rte_epoll_ev, 128, -1);
> +}
> +
> /**
> * Set up environment to talk with a vhost user backend.
> *
> @@ -390,6 +449,7 @@ vhost_user_setup(struct virtio_user_dev *dev)
> {
> int fd;
> int flag;
> + int ret;
> struct sockaddr_un un;
>
> fd = socket(AF_UNIX, SOCK_STREAM, 0);
> @@ -405,13 +465,39 @@ vhost_user_setup(struct virtio_user_dev *dev)
> memset(&un, 0, sizeof(un));
> un.sun_family = AF_UNIX;
> snprintf(un.sun_path, sizeof(un.sun_path), "%s", dev->path);
> - if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
> - PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
> - close(fd);
> - return -1;
> +
> + if (dev->is_server) {
> + static pthread_t fdset_tid;
> +
> + dev->listenfd = fd;
> + dev->epoll_fd = rte_intr_tls_epfd();
> + if (dev->epoll_fd < 0) {
> + PMD_DRV_LOG(ERR, "Can't create epoll file descriptor");
> + return -1;
> + }
> + virtio_user_set_block(dev->epoll_fd, true);
> + if (fdset_tid == 0) {
> + ret = pthread_create(&fdset_tid, NULL, event_dispatch,
> + dev);
> + if (ret < 0) {
> + PMD_DRV_LOG(ERR, "failed to create fdset handling thread");
> + close(fd);
> + close(dev->epoll_fd);
> + return -1;
> + }
> + }
> + return virtio_user_start_server(dev, &un);
> +
> + } else {
> + dev->vhostfd = fd;
> + if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
> + PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
> + close(fd);
> + return -1;
> + }
> + dev->connected = true;
> }
>
> - dev->vhostfd = fd;
> return 0;
> }
>
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> index f90fee9e5..1430b7cbd 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> @@ -142,6 +142,9 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
> {
> uint32_t i;
>
> + if (!dev->connected)
> + return -1;
> +
> for (i = 0; i < dev->max_queue_pairs; ++i)
> dev->ops->enable_qp(dev, i, 0);
>
> @@ -267,21 +270,27 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
> dev->vhostfds = NULL;
> dev->tapfds = NULL;
>
> - if (is_vhost_user_by_type(dev->path)) {
> - dev->ops = &ops_user;
> + if (dev->is_server) {
> + dev->ops = &ops_user;/* server mode only supports vhost user */
> } else {
> - dev->ops = &ops_kernel;
> -
> - dev->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
> - dev->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
> - if (!dev->vhostfds || !dev->tapfds) {
> - PMD_INIT_LOG(ERR, "Failed to malloc");
> - return -1;
> - }
> -
> - for (q = 0; q < dev->max_queue_pairs; ++q) {
> - dev->vhostfds[q] = -1;
> - dev->tapfds[q] = -1;
> + if (is_vhost_user_by_type(dev->path)) {
> + dev->ops = &ops_user;
> + } else {
> + dev->ops = &ops_kernel;
> +
> + dev->vhostfds = malloc(dev->max_queue_pairs *
> + sizeof(int));
> + dev->tapfds = malloc(dev->max_queue_pairs *
> + sizeof(int));
> + if (!dev->vhostfds || !dev->tapfds) {
> + PMD_INIT_LOG(ERR, "Failed to malloc");
> + return -1;
> + }
> +
> + for (q = 0; q < dev->max_queue_pairs; ++q) {
> + dev->vhostfds[q] = -1;
> + dev->tapfds[q] = -1;
> + }
> }
> }
>
> @@ -388,6 +397,13 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
>
> close(dev->vhostfd);
>
> + if (dev->is_server && dev->listenfd >= 0)
> + close(dev->listenfd);
> +
> + if (dev->is_server && dev->epoll_fd >= 0)
> + close(dev->epoll_fd);
> +
> + dev->connected = false;
> if (dev->vhostfds) {
> for (i = 0; i < dev->max_queue_pairs; ++i)
> close(dev->vhostfds[i]);
> @@ -396,6 +412,9 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
> }
>
> free(dev->ifname);
> +
> + if (dev->is_server)
> + unlink(dev->path);
> }
>
> static uint8_t
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.h b/drivers/net/virtio/virtio_user/virtio_user_dev.h
> index 64467b4f9..a8e16d172 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.h
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.h
> @@ -6,6 +6,9 @@
> #define _VIRTIO_USER_DEV_H
>
> #include <limits.h>
> +#include <stdbool.h>
> +#include <sys/epoll.h>
> +#include <rte_interrupts.h>
> #include "../virtio_pci.h"
> #include "../virtio_ring.h"
> #include "vhost.h"
> @@ -13,6 +16,14 @@
> struct virtio_user_dev {
> /* for vhost_user backend */
> int vhostfd;
> + int listenfd; /* listening fd */
> + bool connected; /* connection status */
> +
> + int epoll_fd;
> + struct rte_epoll_event rte_epoll_ev;
> +
> + /* support for server/clinet mode */
> + bool is_server;
>
> /* for vhost_kernel backend */
> char *ifname;
> diff --git a/drivers/net/virtio/virtio_user_ethdev.c b/drivers/net/virtio/virtio_user_ethdev.c
> index 263649006..e60542de5 100644
> --- a/drivers/net/virtio/virtio_user_ethdev.c
> +++ b/drivers/net/virtio/virtio_user_ethdev.c
> @@ -65,8 +65,7 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
> r = recv(dev->vhostfd, buf, 128, MSG_PEEK);
> if (r == 0 || (r < 0 && errno != EAGAIN)) {
> dev->status &= (~VIRTIO_NET_S_LINK_UP);
> - PMD_DRV_LOG(ERR, "virtio-user port %u is down",
> - hw->port_id);
> +
> /* Only client mode is available now. Once the
> * connection is broken, it can never be up
> * again. Besides, this function could be called
> @@ -74,9 +73,15 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
> * callback cannot be unregistered here, set an
> * alarm to do it.
> */
> - rte_eal_alarm_set(1,
> + if (dev->connected) {
> + dev->connected = false;
> + PMD_DRV_LOG(ERR, "virtio-user port %u is down",
> + hw->port_id);
> + rte_eal_alarm_set(1,
> virtio_user_delayed_handler,
> (void *)hw);
> + hw->started = 0;
> + }
> } else {
> dev->status |= VIRTIO_NET_S_LINK_UP;
> }
> @@ -278,12 +283,15 @@ static const char *valid_args[] = {
> VIRTIO_USER_ARG_QUEUE_SIZE,
> #define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
> VIRTIO_USER_ARG_INTERFACE_NAME,
> +#define VIRTIO_USER_ARG_SERVER_MODE "server"
> + VIRTIO_USER_ARG_SERVER_MODE,
> NULL
> };
>
> #define VIRTIO_USER_DEF_CQ_EN 0
> #define VIRTIO_USER_DEF_Q_NUM 1
> #define VIRTIO_USER_DEF_Q_SZ 256
> +#define VIRTIO_USER_DEF_SERVER_MODE 0
>
> static int
> get_string_arg(const char *key __rte_unused,
> @@ -365,6 +373,49 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
> rte_eth_dev_release_port(eth_dev);
> }
>
> +static void
> +virtio_user_server_reconnection(int fd, void *dat)
> +{
> + int ret;
> + int flag;
> + int connectfd;
> + struct virtio_user_dev *dev = dat;
> + struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
> + struct virtio_hw *hw = eth_dev->data->dev_private;
> +
> + if (dev->connected)
> + return;
> +
> + connectfd = accept(fd, NULL, NULL);
> + if (connectfd < 0)
> + return;
> +
> + if (dev->vhostfd >= 0)
> + close(dev->vhostfd);
> +
> + dev->vhostfd = connectfd;
> + flag = fcntl(connectfd, F_GETFD);
> + fcntl(connectfd, F_SETFL, flag & ~O_NONBLOCK);
> +
> + ret = virtio_user_start_device(dev);
> + if (ret < 0)
> + return;
> +
> + if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
> + eth_dev->intr_handle->fd = connectfd;
> + rte_intr_callback_register(eth_dev->intr_handle,
> + virtio_interrupt_handler, eth_dev);
> +
> + if (rte_intr_enable(eth_dev->intr_handle) < 0) {
> + PMD_DRV_LOG(ERR, "interrupt enable failed");
> + return;
> + }
> + }
> +
> + hw->started = 1;
> + dev->connected = true;
> + PMD_INIT_LOG(NOTICE, "virtio user server reconnection succeeds!");
> +}
> /* Dev initialization routine. Invoked once for each virtio vdev at
> * EAL init time, see rte_bus_probe().
> * Returns 0 on success.
> @@ -378,10 +429,12 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
> uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
> uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
> uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
> + uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
> char *path = NULL;
> char *ifname = NULL;
> char *mac_addr = NULL;
> int ret = -1;
> + struct virtio_user_dev *vu_dev = NULL;
>
> kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args);
> if (!kvlist) {
> @@ -445,6 +498,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
> }
> }
>
> + if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
> + if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
> + &get_integer_arg, &server_mode) < 0) {
> + PMD_INIT_LOG(ERR, "error to parse %s",
> + VIRTIO_USER_ARG_SERVER_MODE);
> + goto end;
> + }
> + }
> +
> if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
> if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
> &get_integer_arg, &cq) < 0) {
> @@ -476,6 +538,11 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
> }
>
> hw = eth_dev->data->dev_private;
> + vu_dev = virtio_user_get_dev(hw);
> + if (server_mode == 1)
> + vu_dev->is_server = true;
> + else
> + vu_dev->is_server = false;
> if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq,
> queue_size, mac_addr, &ifname) < 0) {
> PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
> @@ -488,6 +555,16 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
> goto end;
> }
>
> + if (vu_dev->is_server) {
> + vu_dev->rte_epoll_ev.epdata.event = EPOLLIN | EPOLLET;
> + vu_dev->rte_epoll_ev.epdata.cb_fun = virtio_user_server_reconnection;
> + vu_dev->rte_epoll_ev.epdata.cb_arg = vu_dev;
> + ret = rte_epoll_ctl(vu_dev->epoll_fd, EPOLL_CTL_ADD,
> + vu_dev->listenfd, &vu_dev->rte_epoll_ev);
> + if (ret < 0)
> + goto end;
> + }
> +
> /* previously called by rte_pci_probe() for physical dev */
> if (eth_virtio_dev_init(eth_dev) < 0) {
> PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
Hi Jianfeng,
> -----Original Message-----
> From: Tan, Jianfeng
> Sent: Wednesday, March 28, 2018 11:15 PM
> To: Yang, Zhiyong <zhiyong.yang@intel.com>; dev@dpdk.org
> Cc: Wang, Zhihong <zhihong.wang@intel.com>;
> maxime.coquelin@redhat.com; thomas@monjalon.net; Wang, Dong1
> <dong1.wang@intel.com>; Bie, Tiwei <tiwei.bie@intel.com>
> Subject: Re: [PATCH v3 3/4] net/virtio-user: add support for server mode
>
> Hi Zhiyong,
>
> Triggered by the community discussion of thread creation in a library
> (http://dpdk.org/dev/patchwork/patch/36579/), we can think about if
> possible to avoid the pthread creation here.
>
> Just a quick think, it could be feasible. The key idea is to properly set up the
> LSC interrupt for a server mode virtio-user.
> - If the virtio-user device is just probed, we can setup LSC interrupt by
> registering the listen fd to interrupt thread. Whenever a vhost-user
> connection is coming, we will get a chance to read status, at that time, we
> could accept the connection, and report the status is up.
> - And after the connection is setup, we setup LSC interrupt by registering the
> connection fd to interrupt thread; if the connection is broken, we also get a
> change to read status, in which we can switch to the above setting.
>
> How do you think?
>
Very nice idea, which looks reasonable. Let me try it.
Thanks
Zhiyong
> Thanks,
> Jianfeng
>
@@ -378,6 +378,65 @@ vhost_user_sock(struct virtio_user_dev *dev,
return 0;
}
+static void
+virtio_user_set_block(int fd, bool enabled)
+{
+ int f;
+
+ f = fcntl(fd, F_GETFL);
+ if (enabled)
+ fcntl(fd, F_SETFL, f & ~O_NONBLOCK);
+ else
+ fcntl(fd, F_SETFL, f | O_NONBLOCK);
+}
+
+#define MAX_VIRTIO_USER_BACKLOG 128
+static int
+virtio_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un)
+{
+ int ret;
+ int fd = dev->listenfd;
+ int connectfd;
+
+ ret = bind(fd, (struct sockaddr *)un, sizeof(*un));
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again\n",
+ dev->path, strerror(errno));
+ goto err;
+ }
+ ret = listen(fd, MAX_VIRTIO_USER_BACKLOG);
+ if (ret < 0)
+ goto err;
+
+ virtio_user_set_block(fd, true);
+ PMD_DRV_LOG(NOTICE, "virtio user server mode is waiting for connection from vhost user.");
+ while (1) {
+ connectfd = accept(fd, NULL, NULL);
+ if (connectfd >= 0) {
+ dev->connected = true;
+ break;
+ }
+ }
+
+ dev->vhostfd = connectfd;
+ virtio_user_set_block(connectfd, true);
+
+ return 0;
+err:
+ close(dev->epoll_fd);
+ close(dev->listenfd);
+ return -1;
+}
+
+static __attribute__((noreturn)) void *
+event_dispatch(void *arg)
+{
+ struct virtio_user_dev *dev = arg;
+
+ while (1)
+ rte_epoll_wait(dev->epoll_fd, &dev->rte_epoll_ev, 128, -1);
+}
+
/**
* Set up environment to talk with a vhost user backend.
*
@@ -390,6 +449,7 @@ vhost_user_setup(struct virtio_user_dev *dev)
{
int fd;
int flag;
+ int ret;
struct sockaddr_un un;
fd = socket(AF_UNIX, SOCK_STREAM, 0);
@@ -405,13 +465,39 @@ vhost_user_setup(struct virtio_user_dev *dev)
memset(&un, 0, sizeof(un));
un.sun_family = AF_UNIX;
snprintf(un.sun_path, sizeof(un.sun_path), "%s", dev->path);
- if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
- PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
- close(fd);
- return -1;
+
+ if (dev->is_server) {
+ static pthread_t fdset_tid;
+
+ dev->listenfd = fd;
+ dev->epoll_fd = rte_intr_tls_epfd();
+ if (dev->epoll_fd < 0) {
+ PMD_DRV_LOG(ERR, "Can't create epoll file descriptor");
+ return -1;
+ }
+ virtio_user_set_block(dev->epoll_fd, true);
+ if (fdset_tid == 0) {
+ ret = pthread_create(&fdset_tid, NULL, event_dispatch,
+ dev);
+ if (ret < 0) {
+ PMD_DRV_LOG(ERR, "failed to create fdset handling thread");
+ close(fd);
+ close(dev->epoll_fd);
+ return -1;
+ }
+ }
+ return virtio_user_start_server(dev, &un);
+
+ } else {
+ dev->vhostfd = fd;
+ if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
+ PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
+ close(fd);
+ return -1;
+ }
+ dev->connected = true;
}
- dev->vhostfd = fd;
return 0;
}
@@ -142,6 +142,9 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
{
uint32_t i;
+ if (!dev->connected)
+ return -1;
+
for (i = 0; i < dev->max_queue_pairs; ++i)
dev->ops->enable_qp(dev, i, 0);
@@ -267,21 +270,27 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
dev->vhostfds = NULL;
dev->tapfds = NULL;
- if (is_vhost_user_by_type(dev->path)) {
- dev->ops = &ops_user;
+ if (dev->is_server) {
+ dev->ops = &ops_user;/* server mode only supports vhost user */
} else {
- dev->ops = &ops_kernel;
-
- dev->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
- dev->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
- if (!dev->vhostfds || !dev->tapfds) {
- PMD_INIT_LOG(ERR, "Failed to malloc");
- return -1;
- }
-
- for (q = 0; q < dev->max_queue_pairs; ++q) {
- dev->vhostfds[q] = -1;
- dev->tapfds[q] = -1;
+ if (is_vhost_user_by_type(dev->path)) {
+ dev->ops = &ops_user;
+ } else {
+ dev->ops = &ops_kernel;
+
+ dev->vhostfds = malloc(dev->max_queue_pairs *
+ sizeof(int));
+ dev->tapfds = malloc(dev->max_queue_pairs *
+ sizeof(int));
+ if (!dev->vhostfds || !dev->tapfds) {
+ PMD_INIT_LOG(ERR, "Failed to malloc");
+ return -1;
+ }
+
+ for (q = 0; q < dev->max_queue_pairs; ++q) {
+ dev->vhostfds[q] = -1;
+ dev->tapfds[q] = -1;
+ }
}
}
@@ -388,6 +397,13 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
close(dev->vhostfd);
+ if (dev->is_server && dev->listenfd >= 0)
+ close(dev->listenfd);
+
+ if (dev->is_server && dev->epoll_fd >= 0)
+ close(dev->epoll_fd);
+
+ dev->connected = false;
if (dev->vhostfds) {
for (i = 0; i < dev->max_queue_pairs; ++i)
close(dev->vhostfds[i]);
@@ -396,6 +412,9 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
}
free(dev->ifname);
+
+ if (dev->is_server)
+ unlink(dev->path);
}
static uint8_t
@@ -6,6 +6,9 @@
#define _VIRTIO_USER_DEV_H
#include <limits.h>
+#include <stdbool.h>
+#include <sys/epoll.h>
+#include <rte_interrupts.h>
#include "../virtio_pci.h"
#include "../virtio_ring.h"
#include "vhost.h"
@@ -13,6 +16,14 @@
struct virtio_user_dev {
/* for vhost_user backend */
int vhostfd;
+ int listenfd; /* listening fd */
+ bool connected; /* connection status */
+
+ int epoll_fd;
+ struct rte_epoll_event rte_epoll_ev;
+
+ /* support for server/clinet mode */
+ bool is_server;
/* for vhost_kernel backend */
char *ifname;
@@ -65,8 +65,7 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
r = recv(dev->vhostfd, buf, 128, MSG_PEEK);
if (r == 0 || (r < 0 && errno != EAGAIN)) {
dev->status &= (~VIRTIO_NET_S_LINK_UP);
- PMD_DRV_LOG(ERR, "virtio-user port %u is down",
- hw->port_id);
+
/* Only client mode is available now. Once the
* connection is broken, it can never be up
* again. Besides, this function could be called
@@ -74,9 +73,15 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
* callback cannot be unregistered here, set an
* alarm to do it.
*/
- rte_eal_alarm_set(1,
+ if (dev->connected) {
+ dev->connected = false;
+ PMD_DRV_LOG(ERR, "virtio-user port %u is down",
+ hw->port_id);
+ rte_eal_alarm_set(1,
virtio_user_delayed_handler,
(void *)hw);
+ hw->started = 0;
+ }
} else {
dev->status |= VIRTIO_NET_S_LINK_UP;
}
@@ -278,12 +283,15 @@ static const char *valid_args[] = {
VIRTIO_USER_ARG_QUEUE_SIZE,
#define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
VIRTIO_USER_ARG_INTERFACE_NAME,
+#define VIRTIO_USER_ARG_SERVER_MODE "server"
+ VIRTIO_USER_ARG_SERVER_MODE,
NULL
};
#define VIRTIO_USER_DEF_CQ_EN 0
#define VIRTIO_USER_DEF_Q_NUM 1
#define VIRTIO_USER_DEF_Q_SZ 256
+#define VIRTIO_USER_DEF_SERVER_MODE 0
static int
get_string_arg(const char *key __rte_unused,
@@ -365,6 +373,49 @@ virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
rte_eth_dev_release_port(eth_dev);
}
+static void
+virtio_user_server_reconnection(int fd, void *dat)
+{
+ int ret;
+ int flag;
+ int connectfd;
+ struct virtio_user_dev *dev = dat;
+ struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
+ struct virtio_hw *hw = eth_dev->data->dev_private;
+
+ if (dev->connected)
+ return;
+
+ connectfd = accept(fd, NULL, NULL);
+ if (connectfd < 0)
+ return;
+
+ if (dev->vhostfd >= 0)
+ close(dev->vhostfd);
+
+ dev->vhostfd = connectfd;
+ flag = fcntl(connectfd, F_GETFD);
+ fcntl(connectfd, F_SETFL, flag & ~O_NONBLOCK);
+
+ ret = virtio_user_start_device(dev);
+ if (ret < 0)
+ return;
+
+ if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
+ eth_dev->intr_handle->fd = connectfd;
+ rte_intr_callback_register(eth_dev->intr_handle,
+ virtio_interrupt_handler, eth_dev);
+
+ if (rte_intr_enable(eth_dev->intr_handle) < 0) {
+ PMD_DRV_LOG(ERR, "interrupt enable failed");
+ return;
+ }
+ }
+
+ hw->started = 1;
+ dev->connected = true;
+ PMD_INIT_LOG(NOTICE, "virtio user server reconnection succeeds!");
+}
/* Dev initialization routine. Invoked once for each virtio vdev at
* EAL init time, see rte_bus_probe().
* Returns 0 on success.
@@ -378,10 +429,12 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
+ uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
char *path = NULL;
char *ifname = NULL;
char *mac_addr = NULL;
int ret = -1;
+ struct virtio_user_dev *vu_dev = NULL;
kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_args);
if (!kvlist) {
@@ -445,6 +498,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
}
+ if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
+ if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
+ &get_integer_arg, &server_mode) < 0) {
+ PMD_INIT_LOG(ERR, "error to parse %s",
+ VIRTIO_USER_ARG_SERVER_MODE);
+ goto end;
+ }
+ }
+
if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
&get_integer_arg, &cq) < 0) {
@@ -476,6 +538,11 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
hw = eth_dev->data->dev_private;
+ vu_dev = virtio_user_get_dev(hw);
+ if (server_mode == 1)
+ vu_dev->is_server = true;
+ else
+ vu_dev->is_server = false;
if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq,
queue_size, mac_addr, &ifname) < 0) {
PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
@@ -488,6 +555,16 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
goto end;
}
+ if (vu_dev->is_server) {
+ vu_dev->rte_epoll_ev.epdata.event = EPOLLIN | EPOLLET;
+ vu_dev->rte_epoll_ev.epdata.cb_fun = virtio_user_server_reconnection;
+ vu_dev->rte_epoll_ev.epdata.cb_arg = vu_dev;
+ ret = rte_epoll_ctl(vu_dev->epoll_fd, EPOLL_CTL_ADD,
+ vu_dev->listenfd, &vu_dev->rte_epoll_ev);
+ if (ret < 0)
+ goto end;
+ }
+
/* previously called by rte_pci_probe() for physical dev */
if (eth_virtio_dev_init(eth_dev) < 0) {
PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");