From patchwork Wed Jun 19 15:14:31 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Nikos Dragazis X-Patchwork-Id: 54960 X-Patchwork-Delegate: maxime.coquelin@redhat.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 705251C3B3; Wed, 19 Jun 2019 17:15:53 +0200 (CEST) Received: from mx0.arrikto.com (mx0.arrikto.com [212.71.252.59]) by dpdk.org (Postfix) with ESMTP id AF8E21C388 for ; Wed, 19 Jun 2019 17:15:38 +0200 (CEST) Received: from troi.prod.arr (mail.arr [10.99.0.5]) by mx0.arrikto.com (Postfix) with ESMTP id 80BB418200A; Wed, 19 Jun 2019 18:15:38 +0300 (EEST) Received: from localhost.localdomain (unknown [10.89.50.133]) by troi.prod.arr (Postfix) with ESMTPSA id D0C29394; Wed, 19 Jun 2019 18:15:37 +0300 (EEST) From: Nikos Dragazis To: dev@dpdk.org Cc: Maxime Coquelin , Tiwei Bie , Zhihong Wang , Stefan Hajnoczi , Wei Wang , Stojaczyk Dariusz , Vangelis Koukis Date: Wed, 19 Jun 2019 18:14:31 +0300 Message-Id: <1560957293-17294-7-git-send-email-ndragazis@arrikto.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1560957293-17294-1-git-send-email-ndragazis@arrikto.com> References: <1560957293-17294-1-git-send-email-ndragazis@arrikto.com> Subject: [dpdk-dev] [PATCH 06/28] vhost: move vhost-user connection X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" The AF_UNIX transport can accept multiple client connections on a server socket. Each connection instantiates a separate vhost-user device, which is stored as a vhost_user_connection. This behavior is specific to AF_UNIX and other transports may not support N connections per socket endpoint. Move struct vhost_user_connection to trans_af_unix.c and conn_list/conn_mutex into struct af_unix_socket. Signed-off-by: Nikos Dragazis Signed-off-by: Stefan Hajnoczi --- lib/librte_vhost/socket.c | 54 +++--------------------------- lib/librte_vhost/trans_af_unix.c | 72 ++++++++++++++++++++++++++++++++++++---- lib/librte_vhost/vhost.h | 19 ++--------- 3 files changed, 74 insertions(+), 71 deletions(-) diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c index df6d707..976343c 100644 --- a/lib/librte_vhost/socket.c +++ b/lib/librte_vhost/socket.c @@ -341,13 +341,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags) vhost_user_socket_mem_free(vsocket); goto out; } - TAILQ_INIT(&vsocket->conn_list); - ret = pthread_mutex_init(&vsocket->conn_mutex, NULL); - if (ret) { - RTE_LOG(ERR, VHOST_CONFIG, - "error: failed to init connection mutex\n"); - goto out_free; - } vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY; /* @@ -395,7 +388,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags) RTE_LOG(ERR, VHOST_CONFIG, "Postcopy requested but not compiled\n"); ret = -1; - goto out_mutex; + goto out_free; #endif } @@ -403,14 +396,14 @@ rte_vhost_driver_register(const char *path, uint64_t flags) vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT); if (vsocket->reconnect && reconn_tid == 0) { if (vhost_user_reconnect_init() != 0) - goto out_mutex; + goto out_free; } } else { vsocket->is_server = true; } ret = trans_ops->socket_init(vsocket, flags); if (ret < 0) { - goto out_mutex; + goto out_free; } vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket; @@ -418,11 +411,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags) pthread_mutex_unlock(&vhost_user.mutex); return ret; -out_mutex: - if (pthread_mutex_destroy(&vsocket->conn_mutex)) { - RTE_LOG(ERR, VHOST_CONFIG, - "error: failed to destroy connection mutex\n"); - } out_free: vhost_user_socket_mem_free(vsocket); out: @@ -439,51 +427,19 @@ rte_vhost_driver_unregister(const char *path) { int i; int count; - struct vhost_user_connection *conn, *next; if (path == NULL) return -1; -again: pthread_mutex_lock(&vhost_user.mutex); for (i = 0; i < vhost_user.vsocket_cnt; i++) { struct vhost_user_socket *vsocket = vhost_user.vsockets[i]; if (!strcmp(vsocket->path, path)) { - pthread_mutex_lock(&vsocket->conn_mutex); - for (conn = TAILQ_FIRST(&vsocket->conn_list); - conn != NULL; - conn = next) { - next = TAILQ_NEXT(conn, next); - - /* - * If r/wcb is executing, release the - * conn_mutex lock, and try again since - * the r/wcb may use the conn_mutex lock. - */ - if (fdset_try_del(&vhost_user.fdset, - conn->connfd) == -1) { - pthread_mutex_unlock( - &vsocket->conn_mutex); - pthread_mutex_unlock(&vhost_user.mutex); - goto again; - } - - RTE_LOG(INFO, VHOST_CONFIG, - "free connfd = %d for device '%s'\n", - conn->connfd, path); - close(conn->connfd); - vhost_destroy_device(conn->vid); - TAILQ_REMOVE(&vsocket->conn_list, conn, next); - free(conn); - } - pthread_mutex_unlock(&vsocket->conn_mutex); - vsocket->trans_ops->socket_cleanup(vsocket); - - pthread_mutex_destroy(&vsocket->conn_mutex); - vhost_user_socket_mem_free(vsocket); + free(vsocket->path); + free(vsocket); count = --vhost_user.vsocket_cnt; vhost_user.vsockets[i] = vhost_user.vsockets[count]; diff --git a/lib/librte_vhost/trans_af_unix.c b/lib/librte_vhost/trans_af_unix.c index 93d11f7..58fc9e2 100644 --- a/lib/librte_vhost/trans_af_unix.c +++ b/lib/librte_vhost/trans_af_unix.c @@ -15,8 +15,20 @@ #define MAX_VIRTIO_BACKLOG 128 +TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection); + +struct vhost_user_connection { + struct vhost_user_socket *vsocket; + int connfd; + int vid; + + TAILQ_ENTRY(vhost_user_connection) next; +}; + struct af_unix_socket { struct vhost_user_socket socket; /* must be the first field! */ + struct vhost_user_connection_list conn_list; + pthread_mutex_t conn_mutex; int socket_fd; struct sockaddr_un un; }; @@ -131,6 +143,8 @@ send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) static void vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) { + struct af_unix_socket *af_vsocket = + container_of(vsocket, struct af_unix_socket, socket); int vid; size_t size; struct vhost_user_connection *conn; @@ -188,9 +202,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket) goto err_cleanup; } - pthread_mutex_lock(&vsocket->conn_mutex); - TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next); - pthread_mutex_unlock(&vsocket->conn_mutex); + pthread_mutex_lock(&af_vsocket->conn_mutex); + TAILQ_INSERT_TAIL(&af_vsocket->conn_list, conn, next); + pthread_mutex_unlock(&af_vsocket->conn_mutex); fdset_pipe_notify(&vhost_user.fdset); return; @@ -221,6 +235,8 @@ vhost_user_read_cb(int connfd, void *dat, int *remove) { struct vhost_user_connection *conn = dat; struct vhost_user_socket *vsocket = conn->vsocket; + struct af_unix_socket *af_vsocket = + container_of(vsocket, struct af_unix_socket, socket); int ret; ret = vhost_user_msg_handler(conn->vid, connfd); @@ -238,9 +254,9 @@ vhost_user_read_cb(int connfd, void *dat, int *remove) vhost_destroy_device(conn->vid); - pthread_mutex_lock(&vsocket->conn_mutex); - TAILQ_REMOVE(&vsocket->conn_list, conn, next); - pthread_mutex_unlock(&vsocket->conn_mutex); + pthread_mutex_lock(&af_vsocket->conn_mutex); + TAILQ_REMOVE(&af_vsocket->conn_list, conn, next); + pthread_mutex_unlock(&af_vsocket->conn_mutex); free(conn); @@ -512,6 +528,18 @@ static int af_unix_socket_init(struct vhost_user_socket *vsocket, uint64_t flags __rte_unused) { + struct af_unix_socket *af_vsocket = + container_of(vsocket, struct af_unix_socket, socket); + int ret; + + TAILQ_INIT(&af_vsocket->conn_list); + ret = pthread_mutex_init(&af_vsocket->conn_mutex, NULL); + if (ret) { + RTE_LOG(ERR, VHOST_CONFIG, + "error: failed to init connection mutex\n"); + return -1; + } + return create_unix_socket(vsocket); } @@ -520,6 +548,7 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket) { struct af_unix_socket *af_vsocket = container_of(vsocket, struct af_unix_socket, socket); + struct vhost_user_connection *conn, *next; if (vsocket->is_server) { fdset_del(&vhost_user.fdset, af_vsocket->socket_fd); @@ -528,6 +557,37 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket) } else if (vsocket->reconnect) { vhost_user_remove_reconnect(vsocket); } + +again: + pthread_mutex_lock(&af_vsocket->conn_mutex); + for (conn = TAILQ_FIRST(&af_vsocket->conn_list); + conn != NULL; + conn = next) { + next = TAILQ_NEXT(conn, next); + + /* + * If r/wcb is executing, release the + * conn_mutex lock, and try again since + * the r/wcb may use the conn_mutex lock. + */ + if (fdset_try_del(&vhost_user.fdset, + conn->connfd) == -1) { + pthread_mutex_unlock( + &af_vsocket->conn_mutex); + goto again; + } + + RTE_LOG(INFO, VHOST_CONFIG, + "free connfd = %d for device '%s'\n", + conn->connfd, vsocket->path); + close(conn->connfd); + vhost_destroy_device(conn->vid); + TAILQ_REMOVE(&af_vsocket->conn_list, conn, next); + free(conn); + } + pthread_mutex_unlock(&af_vsocket->conn_mutex); + + pthread_mutex_destroy(&af_vsocket->conn_mutex); } static int diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index c74753b..5c3987d 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -404,13 +404,10 @@ struct virtio_net { struct rte_vhost_user_extern_ops extern_ops; } __rte_cache_aligned; -/* The vhost_user, vhost_user_socket, vhost_user_connection, and reconnect - * declarations are temporary measures for moving AF_UNIX code into - * trans_af_unix.c. They will be cleaned up as socket.c is untangled from - * trans_af_unix.c. +/* The vhost_user, vhost_user_socket, and reconnect declarations are temporary + * measures for moving AF_UNIX code into trans_af_unix.c. They will be cleaned + * up as socket.c is untangled from trans_af_unix.c. */ -TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection); - /* * Every time rte_vhost_driver_register() is invoked, an associated * vhost_user_socket struct will be created. @@ -421,8 +418,6 @@ TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection); * struct. */ struct vhost_user_socket { - struct vhost_user_connection_list conn_list; - pthread_mutex_t conn_mutex; char *path; bool is_server; bool reconnect; @@ -453,14 +448,6 @@ struct vhost_user_socket { struct vhost_transport_ops const *trans_ops; }; -struct vhost_user_connection { - struct vhost_user_socket *vsocket; - int connfd; - int vid; - - TAILQ_ENTRY(vhost_user_connection) next; -}; - #define MAX_VHOST_SOCKET 1024 struct vhost_user { struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET];